submit_eA_xsec_validation_mc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #
4 #-----------------------------------------------------------------------------------------------------------
5 # Submit jobs to generate data needed for validating GENIE (e,e') differential cross-section modelling.
6 #
7 # Syntax:
8 # perl submit_eA_xsec_validation_mc_jobs.pl <options>
9 #
10 # Options:
11 # --version : GENIE version number
12 # --run : runs to submit (eg --run 100601200200 / --run 100601200200,100801600680 / --run all / --run 12C)
13 # [--model-enum] : physics model enumeration, default: 0
14 # [--nsubruns] : number of subruns per run, default: 1
15 # @ [--Emin] : minimum energy (eg `--run 56Fe --Emin 1.500' will only run the standard iron jobs above 1.5 GeV)
16 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
17 # [--production] : production name, default: <model>_<version>
18 # [--cycle] : cycle in current production, default: 01
19 # [--use-valgrind] : default: off
20 # [--batch-system] : <PBS, LSF, slurm, none>, default: PBS
21 # [--queue] : default: prod
22 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
23 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
24 #
25 # Run number key: IZZZAAAEEEEEMxx
26 #
27 # I : probe (1:e-, 2:e+)
28 # ZZZ : target Z (eg, 001:H1, 026:Fe56)
29 # AAA : target A (eg: 056:Fe56, 238:U)
30 # EEEEE : energy used in MeV (eg 00680->0.68GeV, 02015->2.015GeV etc)
31 # M : physics model enumeration, 0-9
32 # xx : sub-run ID, 00-99, 100k events each
33 #
34 #
35 # Author:
36 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
37 # University of Liverpool & STFC Rutherford Appleton Laboratory
38 #
39 # Copyright:
40 # Copyright (c) 2003-2020, The GENIE Collaboration
41 # For the full text of the license visit http://copyright.genie-mc.org
42 #-----------------------------------------------------------------------------------------------------------
43 #
44 
45 use File::Path;
46 
47 # inputs
48 #
49 $iarg=0;
50 foreach (@ARGV) {
51  if($_ eq '--Emin') { $Emin = $ARGV[$iarg+1]; }
52  if($_ eq '--nsubruns') { $nsubruns = $ARGV[$iarg+1]; }
53  if($_ eq '--run') { $run = $ARGV[$iarg+1]; }
54  if($_ eq '--model-enum') { $model_enum = $ARGV[$iarg+1]; }
55  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
56  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
57  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
58  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
59  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
60  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
61  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
62  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
63  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
64  $iarg++;
65 }
66 die("** Aborting [Undefined GENIE version. Use the --version option]")
67 unless defined $genie_version;
68 die("** Aborting [You need to specify which runs to submit. Use the --run option]")
69 unless defined $run;
70 
71 $model_enum = "0" unless defined $model_enum;
72 $nsubruns = 1 unless defined $nsubruns;
73 $use_valgrind = 0 unless defined $use_valgrind;
74 $arch = "SL6.x86_64" unless defined $arch;
75 $production = "$model_enum\_$genie_version" unless defined $production;
76 $cycle = "01" unless defined $cycle;
77 $batch_system = "PBS" unless defined $batch_system;
78 $queue = "prod" unless defined $queue;
79 $softw_topdir = "/opt/ppd/t2k/softw/GENIE" unless defined $softw_topdir;
80 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
81 $time_limit = "60:00:00";
82 $xspl_file = "$softw_topdir/data/job_inputs/xspl/gxspl-eA-$genie_version.xml";
83 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
84 $jobs_dir = "$jobs_topdir/eA-$production\_$cycle";
85 $mcseed = 210921029;
86 $nev_per_subrun = 100000;
87 
88 #
89 # MC runs required for generating predictions for all datasets described
90 # data/validation/eA/xsec/differential/qe/datasets.txt
91 #
92 @std_runnu =
93 (
94 # 2D
95 # ...
96 # ... to be added
97 # ...
98 # 3H
99 # ...
100 # ... to be added
101 # ...
102 # 3He
103 # ...
104 # ... to be added
105 # ...
106 # 4He
107 # ...
108 # ... to be added
109 # ...
110 # 12C
111  100601200160, # (e-,e-')12C, 0.160 GeV
112  100601200200, # (e-,e-')12C, 0.200 GeV
113  100601200240, # (e-,e-')12C, 0.240 GeV
114  100601200280, # (e-,e-')12C, 0.280 GeV
115  100601200320, # (e-,e-')12C, 0.320 GeV
116  100601200361, # (e-,e-')12C, 0.361 GeV
117  100601200400, # (e-,e-')12C, 0.400 GeV
118  100601200440, # (e-,e-')12C, 0.440 GeV
119  100601200480, # (e-,e-')12C, 0.480 GeV
120  100601200500, # (e-,e-')12C, 0.500 GeV
121  100601200519, # (e-,e-')12C, 0.519 GeV
122  100601200560, # (e-,e-')12C, 0.560 GeV
123  100601200620, # (e-,e-')12C, 0.620 GeV
124  100601200680, # (e-,e-')12C, 0.680 GeV
125  100601200730, # (e-,e-')12C, 0.730 GeV
126  100601200961, # (e-,e-')12C, 0.961 GeV
127  100601201108, # (e-,e-')12C, 1.108 GeV
128  100601201300, # (e-,e-')12C, 1.300 GeV
129  100601201500, # (e-,e-')12C, 1.500 GeV
130  100601201650, # (e-,e-')12C, 1.650 GeV
131  100601201930, # (e-,e-')12C, 1.930 GeV
132  100601202000, # (e-,e-')12C, 2.000 GeV
133  100601202015, # (e-,e-')12C, 2.015 GeV
134  100601202020, # (e-,e-')12C, 2.020 GeV
135  100601202130, # (e-,e-')12C, 2.130 GeV
136  100601202500, # (e-,e-')12C, 2.500 GeV
137  100601202700, # (e-,e-')12C, 2.700 GeV
138  100601203188, # (e-,e-')12C, 3.188 GeV
139  100601203595, # (e-,e-')12C, 3.595 GeV
140  100601203605, # (e-,e-')12C, 3.605 GeV
141  100601204045, # (e-,e-')12C, 4.045 GeV
142  100601204212, # (e-,e-')12C, 4.212 GeV
143  100601205120, # (e-,e-')12C, 5.120 GeV
144 # 16O
145  100801600700, # (e-,e-')16O, 0.700 GeV
146  100801600737, # (e-,e-')16O, 0.737 GeV
147  100801600880, # (e-,e-')16O, 0.880 GeV
148  100801601080, # (e-,e-')16O, 1.080 GeV
149  100801601200, # (e-,e-')16O, 1.200 GeV
150  100801601500, # (e-,e-')16O, 1.500 GeV
151 # ...
152 # ... to be added
153 # ...
154 # 27Al
155 # ...
156 # ... to be added
157 # ...
158 # 40Ca
159 # ...
160 # ... to be added
161 # ...
162 # 48Ca
163 # ...
164 # ... to be added
165 # ...
166 # 56Fe
167  102605600120, # (e-,e-')56Fe, 0.120 GeV
168  102605600155, # (e-,e-')56Fe, 0.155 GeV
169  102605600160, # (e-,e-')56Fe, 0.160 GeV
170  102605600174, # (e-,e-')56Fe, 0.174 GeV
171  102605600192, # (e-,e-')56Fe, 0.192 GeV
172  102605600200, # (e-,e-')56Fe, 0.200 GeV
173  102605600206, # (e-,e-')56Fe, 0.206 GeV
174  102605600221, # (e-,e-')56Fe, 0.221 GeV
175  102605600236, # (e-,e-')56Fe, 0.236 GeV
176  102605600240, # (e-,e-')56Fe, 0.240 GeV
177  102605600252, # (e-,e-')56Fe, 0.252 GeV
178  102605600271, # (e-,e-')56Fe, 0.271 GeV
179  102605600280, # (e-,e-')56Fe, 0.280 GeV
180  102605600293, # (e-,e-')56Fe, 0.293 GeV
181  102605600312, # (e-,e-')56Fe, 0.312 GeV
182  102605600320, # (e-,e-')56Fe, 0.320 GeV
183  102605600333, # (e-,e-')56Fe, 0.333 GeV
184  102605600340, # (e-,e-')56Fe, 0.340 GeV
185  102605600360, # (e-,e-')56Fe, 0.360 GeV
186  102605600400, # (e-,e-')56Fe, 0.400 GeV
187  102605600440, # (e-,e-')56Fe, 0.440 GeV
188  102605600480, # (e-,e-')56Fe, 0.480 GeV
189  102605600560, # (e-,e-')56Fe, 0.560 GeV
190  102605600620, # (e-,e-')56Fe, 0.620 GeV
191  102605600900, # (e-,e-')56Fe, 0.900 GeV
192  102605600961, # (e-,e-')56Fe, 0.961 GeV
193  102605601100, # (e-,e-')56Fe, 1.100 GeV
194  102605601108, # (e-,e-')56Fe, 1.108 GeV
195  102605601250, # (e-,e-')56Fe, 1.250 GeV
196  102605601299, # (e-,e-')56Fe, 1.299 GeV
197  102605601500, # (e-,e-')56Fe, 1.500 GeV
198  102605601650, # (e-,e-')56Fe, 1.650 GeV
199  102605602015, # (e-,e-')56Fe, 2.015 GeV
200  102605602020, # (e-,e-')56Fe, 2.020 GeV
201  102605602700, # (e-,e-')56Fe, 2.700 GeV
202  102605603188, # (e-,e-')56Fe, 3.188 GeV
203  102605603300, # (e-,e-')56Fe, 3.300 GeV
204  102605603595, # (e-,e-')56Fe, 3.595 GeV
205  102605603600, # (e-,e-')56Fe, 3.600 GeV
206  102605603605, # (e-,e-')56Fe, 3.605 GeV
207  102605603900, # (e-,e-')56Fe, 3.900 GeV
208  102605603995, # (e-,e-')56Fe, 3.995 GeV
209  102605604045, # (e-,e-')56Fe, 4.045 GeV
210  102605604212, # (e-,e-')56Fe, 4.212 GeV
211  102605604300, # (e-,e-')56Fe, 4.300 GeV
212  102605605120 # (e-,e-')56Fe, 5.120 GeV
213 # ...
214 # ... to be added
215 # ...
216 # 197Au
217 # ...
218 # ... to be added
219 # ...
220 # 208Pb
221 # ...
222 # ... to be added
223 # ...
224 # 238U
225 # ...
226 # ... to be added
227 # ...
228 );
229 
230 @runnu = ();
231 if($run eq "all") {
232  push(@runnu, @std_runnu);
233 }
234 else {
235  my @a = split(',', $run);
236  push(@runnu, @a);
237 }
238 
239 # make the jobs directory
240 #
241 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
242 
243 #
244 # submit event generation jobs
245 #
246 
247 # run loop
248 foreach(@runnu) {
249  my $curr_runnu = $_;
250  print "** submitting event generation run: $curr_runnu \n";
251 
252  # match probe,Z,A,E from current run
253  my ($m_probe, $m_Z, $m_A, $m_E) = $curr_runnu =~ /([0-9]{1})([0-9]{3})([0-9]{3})([0-9]{5})/;
254 
255  # convert to GENIE-expected probe and target PDG codes and energy
256  $tgt_pdg = 1000000000 + $m_Z*10000 + $m_A*10;
257  $E = $m_E/1000.;
258 
259  if ($m_probe == 1) { $probe_pdg = 11; }
260  elsif ($m_probe == 2) { $probe_pdg = -11; }
261  else { die; }
262 
263  # submit subruns
264  for($isubrun = 0; $isubrun < $nsubruns; $isubrun++) {
265 
266  # Run number key: IZZZAAAEEEEEMxx
267  $curr_subrunnu = 1000 * $curr_runnu + 100 * $model_enum + $isubrun;
268 
269  $grep_pipe = "grep -B 20 -A 30 -i \"warn\\|error\\|fatal\"";
270  $fntemplate = "$jobs_dir/eA-$curr_subrunnu";
271  $curr_seed = $mcseed + $isubrun;
272  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
273  $evgen_opt = "-n $nev_per_subrun -e $E -p $probe_pdg -t $tgt_pdg -r $curr_subrunnu --seed $curr_seed --cross-sections $xspl_file --event-generator-list EM";
274  $evgen_cmd = "gevgen $evgen_opt | $grep_pipe &> $fntemplate.evgen.log";
275  $conv_cmd = "gntpc -f gst -i gntp.$curr_subrunnu.ghep.root | $grep_pipe &> $fntemplate.conv.log";
276 
277  print "@@ exec: $evgen_cmd \n";
278 
279  #
280  # submit
281  #
282 
283  # PBS case
284  if($batch_system eq 'PBS') {
285  $batch_script = "$fntemplate.pbs";
286  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
287  print PBS "#!/bin/bash \n";
288  print PBS "#PBS -N nucl-$curr_subrunnu \n";
289  print PBS "#PBS -l cput=$time_limit \n";
290  print PBS "#PBS -o $fntemplate.pbsout.log \n";
291  print PBS "#PBS -e $fntemplate.pbserr.log \n";
292  print PBS "source $genie_setup \n";
293  print PBS "cd $jobs_dir \n";
294  print PBS "$evgen_cmd \n";
295  print PBS "$conv_cmd \n";
296  close(PBS);
297  `qsub -q $queue $batch_script`;
298  } # PBS
299 
300  # LSF case
301  if($batch_system eq 'LSF') {
302  $batch_script = "$fntemplate.sh";
303  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
304  print LSF "#!/bin/bash \n";
305  print LSF "#BSUB-j nucl-$curr_subrunnu \n";
306  print LSF "#BSUB-q $queue \n";
307  print LSF "#BSUB-c $time_limit \n";
308  print LSF "#BSUB-o $fntemplate.lsfout.log \n";
309  print LSF "#BSUB-e $fntemplate.lsferr.log \n";
310  print LSF "source $genie_setup \n";
311  print LSF "cd $jobs_dir \n";
312  print LSF "$evgen_cmd \n";
313  print LSF "$conv_cmd \n";
314  close(LSF);
315  `bsub < $batch_script`;
316  } # LSF
317 
318  # slurm case
319  if($batch_system eq 'slurm') {
320  my $time_lim = `sinfo -h -p batch -o %l`;
321  my ($days, $hours, $remainder) = $time_lim =~ /([0]+)-([0-9]+):(.*)/;
322  my $newhours = $days * 24 + $hours;
323  my $new_time_lim = "$newhours:$remainder";
324  $time_limit = $new_time_lim lt $time_limit ? $new_time_lim : $time_limit;
325  $batch_script = "$fntemplate.sh";
326  open(SLURM, ">$batch_script") or die("Can not create the SLURM batch script");
327  print SLURM "#!/bin/bash \n";
328  print SLURM "#SBATCH-p $queue \n";
329  print SLURM "#SBATCH-o $fntemplate.lsfout.log \n";
330  print SLURM "#SBATCH-e $fntemplate.lsferr.log \n";
331  print SLURM "#SBATCH-t $time_limit \n";
332  print SLURM "source $genie_setup \n";
333  print SLURM "cd $jobs_dir \n";
334  print SLURM "$evgen_cmd \n";
335  print SLURM "$conv_cmd \n";
336  close(SLURM);
337  `sbatch --job-name=nucl-$curr_subrunnu $batch_script`;
338  } # slurm
339 
340  # no batch system, run jobs interactively
341  if($batch_system eq 'none') {
342  system("source $genie_setup; cd $jobs_dir; $evgen_cmd; $conv_cmd");
343  } # interactive mode
344 
345  } # loop over subruns
346 
347 } # loop over runs
348 
Generated by   doxygen 1.8.11