run_everything.sh
Go to the documentation of this file.
1 #! /usr/bin/env bash
2 
3 HORN=$1
4 FIRST=$2
5 NPOT=$3
6 OFFAXIS=$4
7 FLUX=$5
8 TEST=$6
9 
10 TIME_START=`date +%s`
11 
12 if [ "${HORN}" != "FHC" ] && [ "${HORN}" != "RHC" ]; then
13 echo "Invalid beam mode ${HORN}"
14 echo "Must be FHC or RHC"
15 kill -INT $$
16 fi
17 
18 MODE="neutrino"
19 RHC=""
20 if [ "${HORN}" = "RHC" ]; then
21 MODE="antineutrino"
22 RHC=" --rhc"
23 fi
24 
25 if [ "${NPOT}" = "" ]; then
26 echo "POT not specified, using 1E16"
27 NPOT=1E16
28 fi
29 
30 if [ "${FIRST}" = "" ]; then
31 echo "First run number not specified, using 0"
32 FIRST=0
33 fi
34 
35 if [ "${OFFAXIS}" = "" ]; then
36 echo "Off axis position not specified, assuming on-axis"
37 OFFAXIS=0
38 fi
39 
40 if [ "${FLUX}" = "" ]; then
41 echo "Flux not specified, using dk2nu"
42 FLUX="dk2nu"
43 fi
44 
45 if [ "${FLUX}" = "dk2nu" ]; then
46 FLUXOPT="--dk2nu"
47 FLUXDIR="/pnfs/dune/persistent/users/ljf26/fluxfiles/g4lbne/v3r5p4/QGSP_BERT"
48 fi
49 if [ "${FLUX}" = "gsimple" ]; then
50 FLUXOPT=""
51 FLUXDIR="/pnfs/dune/persistent/users/dbrailsf/flux/nd/gsimple/v2_8_6d"
52 fi
53 
54 CP="ifdh cp"
55 if [ "${TEST}" = "test" ]; then
56 echo "Test mode"
57 PROCESS=0
58 mkdir -p test
59 cd test
60 fi
61 
62 # ifdhc doen't have a mkdir -p equivalent, which is fine
63 # as long as you always remember to include this convenient function in your scripts
64 ifdh_mkdir_p() {
65  local dir=$1
66  local force=$2
67  if [ `ifdh ls $dir 0 $force | wc -l` -gt 0 ]
68  then
69  : # we're done
70  else
71  ifdh_mkdir_p `dirname $dir` $force
72  ifdh mkdir $dir $force
73  fi
74 }
75 
76 OADIR="${OFFAXIS}m"
77 if [ "${HORN}" = "RHC" ]; then
78 OADIR="${OFFAXIS}mRHC"
79 fi
80 
81 if [ "${FLUX}" = "gsimple" ] && [ "${HORN}" = "FHC" ]; then
82 OADIR="${OFFAXIS}mgsimple"
83 fi
84 
85 RUNNO=$((${PROCESS}+${FIRST}))
86 RNDSEED=$((1000000*${OFFAXIS}+${RUNNO}+1000000))
87 
88 # 5E16 is about 15000 events on-axis which runs in ~6 hours
89 NEVENTS="-e ${NPOT}" # -n XXXX number of events, -e XE16 for POT
90 
91 RDIR=$((${RUNNO} / 1000))
92 if [ ${RUNNO} -lt 10000 ]; then
93 RDIR=0$((${RUNNO} / 1000))
94 fi
95 
96 GEOMETRY="MPD_SPY_LAr"
97 TOPVOL="volArgonCubeActive"
98 
99 TARDIR="/pnfs/dune/persistent/users/LBL_TDR/sw_tarballs"
100 OUTDIR="/pnfs/dune/persistent/users/marshalc/nd_offaxis/v7"
101 
102 # Don't try over and over again to copy a file when it isn't going to work
103 export IFDH_CP_UNLINK_ON_ERROR=1
104 export IFDH_CP_MAXRETRIES=1
105 export IFDH_DEBUG=0
106 
107 ##################################################
108 
109 # Setup UPS and required products
110 echo "Setting up software"
111 source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
112 setup ifdhc
113 setup dk2nugenie v01_06_01f -q debug:e15
114 setup genie_xsec v2_12_10 -q DefaultPlusValenciaMEC
115 setup genie_phyopt v2_12_10 -q dkcharmtau
116 setup geant4 v4_10_3_p01b -q e15:prof
117 
118 # edep-sim needs to know where a certain GEANT .cmake file is...
119 G4_cmake_file=`find ${GEANT4_FQ_DIR}/lib64 -name 'Geant4Config.cmake'`
120 export Geant4_DIR=`dirname $G4_cmake_file`
121 
122 # edep-sim needs to have the GEANT bin directory in the path
123 export PATH=$PATH:$GEANT4_FQ_DIR/bin
124 
125 ##################################################
126 # Get the binaries & other files that are needed
127 ${CP} ${TARDIR}/sim_inputs.tar.gz sim_inputs.tar.gz
128 ${CP} ${TARDIR}/edep-sim.tar.gz edep-sim.tar.gz
129 ${CP} ${TARDIR}/nusystematics.tar.gz nusystematics.tar.gz
130 ${CP} ${TARDIR}/nusyst_inputs.tar.gz nusyst_inputs.tar.gz
131 ${CP} ${TARDIR}/DUNE_ND_GeoEff.tar.gz DUNE_ND_GeoEff.tar.gz
132 
133 
134 tar -xzf sim_inputs.tar.gz
135 tar -xzf edep-sim.tar.gz
136 tar -xzf nusystematics.tar.gz
137 tar -xzf nusyst_inputs.tar.gz
138 tar -xzf DUNE_ND_GeoEff.tar.gz
139 mv sim_inputs/* ${PWD}
140 
141 # Get flux files to local node
142 # dk2nu files: /pnfs/dune/persistent/users/ljf26/fluxfiles/g4lbne/v3r5p4/QGSP_BERT/OptimizedEngineeredNov2017/neutrino/flux/dk2nu
143 # gsimple files: /pnfs/dune/persistent/users/dbrailsf/flux/nd/gsimple/v2_8_6d/OptimizedEngineeredNov2017/neutrino/
144 chmod +x copy_dune_ndtf_flux
145 ./copy_dune_ndtf_flux --top ${FLUXDIR} --outpath local_flux_files --flavor ${MODE} --base OptimizedEngineeredNov2017 --maxmb=300 ${FLUXOPT}
146 
147 # GENIE for some reason doesn't recognize *.dk2nu.root as dk2nu format, but it works if dk2nu is at the front?
148 if [ "${FLUX}" = "dk2nu" ]; then
149 cd local_flux_files
150 for f in *.dk2nu.root
151 do
152  mv "$f" "dk2nu_$f"
153 done
154 cd ..
155 fi
156 
157 ##################################################
158 
159 # Modify GNuMIFlux.xml to the specified off-axis position
160 sed -i "s/<beampos> ( 0.0, 0.05387, 6.66 )/<beampos> ( ${OFFAXIS}, 0.05387, 6.66 )/g" GNuMIFlux.xml
161 
162 export GXMLPATH=${PWD}:${GXMLPATH}
163 export GNUMIXML="GNuMIFlux.xml"
164 
165 # Run GENIE
166 echo "Running gevgen"
167 TIME_GENIE=`date +%s`
168 gevgen_fnal \
169  -f local_flux_files/${FLUX}*,DUNEND \
170  -g ${GEOMETRY}.gdml \
171  -t ${TOPVOL} \
172  -L cm -D g_cm3 \
173  ${NEVENTS} \
174  --seed ${RNDSEED} \
175  -r ${RNDSEED} \
176  -o ${MODE} \
177  --message-thresholds Messenger_production.xml \
178  --cross-sections ${GENIEXSECPATH}/gxspl-FNALsmall.xml \
179  --event-record-print-level 0 \
180  --event-generator-list Default+CCMEC
181  #-m ${GEOMETRY}.${TOPVOL}.maxpl.xml \
182 
183 ##################################################
184 
185 # Convert the genie output to rootracker
186 
187 export LD_LIBRARY_PATH=${PWD}/edep-sim/edep-gcc-6.4.0-x86_64-pc-linux-gnu/lib:${LD_LIBRARY_PATH}
188 export PATH=${PWD}/edep-sim/edep-gcc-6.4.0-x86_64-pc-linux-gnu/bin:${PATH}
189 
190 echo "Running gntpc"
191 TIME_ROOTRACKER=`date +%s`
192 ${CP} ${MODE}.${RNDSEED}.ghep.root input_file.ghep.root
193 gntpc -i input_file.ghep.root -f rootracker \
194  --event-record-print-level 0 \
195  --message-thresholds Messenger_production.xml
196 
197 # edep-sim wants number of events, but we are doing POT so the files will be slightly different
198 # get the number of events from the GENIE files to pass it to edep-sim
199 NPER=$(echo "std::cout << gtree->GetEntries() << std::endl;" | genie -l -b input_file.ghep.root 2>/dev/null | tail -1)
200 
201 ##################################################
202 
203 # Run edep-sim
204 echo "Running edep-sim with ${NPER} events."
205 TIME_EDEPSIM=`date +%s`
206 edep-sim \
207  -C \
208  -g ${GEOMETRY}.gdml \
209  -o edep.${RNDSEED}.root \
210  -u \
211  -e ${NPER} \
212  dune-nd.mac
213 
214 ##################################################
215 
216 # The MakeProject in dumpTree.py won't work if edep-sim is in the library path for reasons unknown
217 # This is a hack, please avert your eyes if you don't want to see my garbage
218 unset LD_LIBRARY_PATH
219 setup ifdhc
220 setup dk2nugenie v01_06_01f -q debug:e15
221 setup genie_xsec v2_12_10 -q DefaultPlusValenciaMEC
222 setup genie_phyopt v2_12_10 -q dkcharmtau
223 setup geant4 v4_10_3_p01b -q e15:prof
224 
225 # Add nusystematics to the paths
226 export LD_LIBRARY_PATH=${PWD}/nusystematics/build/Linux/lib:${LD_LIBRARY_PATH}
227 export LD_LIBRARY_PATH=${PWD}/nusystematics/build/nusystematics/artless:${LD_LIBRARY_PATH}
228 export FHICL_FILE_PATH=${PWD}/nusystematics/nusystematics/fcl:${FHICL_FILE_PATH}
229 
230 # add pyGeoEff to pythonpath, and libgeoEff to LD_LIBRARY_PATH
231 export PYTHONPATH=${PWD}/DUNE_ND_GeoEff/lib/:${PYTHONPATH}
232 export LD_LIBRARY_PATH=${PWD}/DUNE_ND_GeoEff/lib:${LD_LIBRARY_PATH}
233 
234 # Run dumpTree to make a root file, you can start reading again if you averted your eyes before
235 TIME_DUMPTREE=`date +%s`
236 python dumpTree.py --infile edep.${RNDSEED}.root --outfile ${HORN}.${RNDSEED}.root --seed ${RNDSEED}
237 
238 # Run CAFMaker
239 TIME_CAFMAKER=`date +%s`
240 ./makeCAF --infile ${HORN}.${RNDSEED}.root --gfile ${MODE}.${RNDSEED}.ghep.root --outfile ${HORN}.${RNDSEED}.CAF.root --fhicl ./fhicl.fcl --seed ${RNDSEED} ${RHC} --oa ${OFFAXIS}
241 
242 ##################################################
243 # Copy the output files
244 echo "It's copy time, here are the files that I have:"
245 TIME_COPY=`date +%s`
246 
247 ifdh_mkdir_p ${OUTDIR}/genie/${OADIR}/${RDIR}
248 ifdh_mkdir_p ${OUTDIR}/dump/${OADIR}/${RDIR}
249 ifdh_mkdir_p ${OUTDIR}/CAF/${OADIR}/${RDIR}
250 ifdh_mkdir_p ${OUTDIR}/edep/${OADIR}/${RDIR}
251 
252 # GENIE, this is usually small and good idea to save
253 ${CP} ${MODE}.${RNDSEED}.ghep.root ${OUTDIR}/genie/${OADIR}/${RDIR}/${HORN}.${RNDSEED}.ghep.root
254 
255 # G4/edep-sim file is HUGE and probably we can't save it
256 #${CP} edep.${RNDSEED}.root ${OUTDIR}/edep/${OADIR}/${RDIR}/${HORN}.${RNDSEED}.edep.root
257 
258 # "dump" file is useful for various analyses
259 ${CP} ${HORN}.${RNDSEED}.root ${OUTDIR}/dump/${OADIR}/${RDIR}/${HORN}.${RNDSEED}.dump.root
260 
261 ${CP} ${HORN}.${RNDSEED}.CAF.root ${OUTDIR}/CAF/${OADIR}/${RDIR}/${HORN}.${RNDSEED}.CAF.root
262 
263 TIME_END=`date +%s`
264 # Print out a single thing that says the time of each step
265 TIME_S=$((${TIME_GENIE}-${TIME_START}))
266 TIME_G=$((${TIME_ROOTRACKER}-${TIME_GENIE}))
267 TIME_R=$((${TIME_EDEPSIM}-${TIME_ROOTRACKER}))
268 TIME_E=$((${TIME_DUMPTREE}-${TIME_EDEPSIM}))
269 TIME_D=$((${TIME_CAFMAKER}-${TIME_DUMPTREE}))
270 TIME_M=$((${TIME_COPY}-${TIME_CAFMAKER}))
271 TIME_C=$((${TIME_END}-${TIME_COPY}))
272 echo "Start-up time: ${TIME_S}"
273 echo "gevgen time: ${TIME_G}"
274 echo "gntpc time: ${TIME_R}"
275 echo "edep-sim time: ${TIME_E}"
276 echo "dumpTree time: ${TIME_D}"
277 echo "makeCAF time: ${TIME_M}"
278 echo "Copy time: ${TIME_C}"
Generated by   doxygen 1.8.11