submit_standard_neutrino_mc_test_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #---------------------------------------------------------------------------------------------------------------------
4 # Submit standard neutrino event generation jobs for GENIE release validation
5 # The outputs can be compared with outputs from past releases using the GENIE/Generator gevcomp app.
6 # Sanity checks can be performed using the GENIE/Generator gevscan app.
7 #
8 # Syntax:
9 # shell% perl submit_standard_neutrino_mc_test_jobs.pl <options>
10 #
11 # Options:
12 # --version : GENIE version number
13 # --run : Comma separated list of run numbers
14 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
15 # [--production] : default: routine_validation
16 # [--cycle] : default: 01
17 # [--ref-samples] : Path for reference samples, default: no reference samples / no plots will be generated
18 # [--use-valgrind] : default: off
19 # [--batch-system] : <PBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: PBS
20 # [--queue] : default: prod
21 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
22 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
23 #
24 #
25 # SAMPLES:
26 #......................................................................
27 # run | nev | init state | energy | processes
28 # nu. | | | (GeV) | enabled
29 #......................................................................
30 # 1000 | 100k | numu + n | 0.5 | all
31 # 1001 | 100k | numu + n | 1 | all
32 # 1002 | 100k | numu + n | 5 | all
33 # 1003 | 100k | numu + n | 50 | all
34 # 1101 | 100k | numubar + p | 1 | all
35 # 1102 | 100k | numubar + p | 5 | all
36 # 1103 | 100k | numubar + p | 50 | all
37 # 2001 | 100k | numu + Fe56 | 1 | all
38 # 2002 | 100k | numu + Fe56 | 5 | all
39 # 2003 | 100k | numu + Fe56 | 50 | all
40 # 2101 | 100k | numubar + Fe56 | 1 | all
41 # 2102 | 100k | numubar + Fe56 | 5 | all
42 # 2103 | 100k | numubar + Fe56 | 50 | all
43 # 9001 | 100k | numu + Fe56 | 5 | DIS charm
44 # 9002 | 100k | numu + Fe56 | 5 | QEL charm
45 # 9101 | 100k | numu + Fe56 | 2 | COH CC+NC
46 # 9201 | 100k | nue + Fe56 | 1 | ve elastic
47 # 9202 | 100k | numu + Fe56 | 1 | ve elastic
48 # 9203 | 50k | numu + Fe56 | 20 | IMD
49 # 9204 | 50k | nuebar + Fe56 | 20 | IMD (annihilation)
50 #......................................................................
51 #
52 #
53 # Examples:
54 # shell% perl submit_standard_neutrino_mc_test_jobs.pl \
55 # --production 2.5.1_prelease_tests --cycle 01 --version v2.5.1 --run 1001
56 # shell% perl submit_standard_neutrino_mc_test_jobs.pl \
57 # --production 2.5.1_prelease_tests --cycle 01 --version v2.5.1 --run 1000,1001,9203
58 # shell% perl submit_standard_neutrino_mc_test_jobs.pl \
59 # --production 2.5.1_prelease_tests --cycle 01 --version v2.5.1 --run all
60 #
61 # Tested at the RAL/PPD Tier2 PBS batch farm.
62 #
63 # Costas Andreopoulos <constantinos.andreopoulos \at cern.ch>
64 # STFC, Rutherford Appleton Lab
65 #---------------------------------------------------------------------------------------------------------------------
66 #
67 #
68 
69 use File::Path;
70 
71 # inputs
72 #
73 $iarg=0;
74 foreach (@ARGV) {
75  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
76  if($_ eq '--run') { $runnu = $ARGV[$iarg+1]; }
77  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
78  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
79  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
80  if($_ eq '--ref-samples') { $ref_sample_path = $ARGV[$iarg+1]; }
81  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
82  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
83  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
84  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
85  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
86  $iarg++;
87 }
88 die("** Aborting [Undefined benchmark runs #. Use the --run option]")
89 unless defined $runnu;
90 die("** Aborting [Undefined GENIE version. Use the --version option]")
91 unless defined $genie_version;
92 
93 $use_valgrind = 0 unless defined $use_valgrind;
94 $arch = "SL6.x86_64" unless defined $arch;
95 $production = "routine_validation" unless defined $production;
96 $cycle = "01" unless defined $cycle;
97 $batch_system = "PBS" unless defined $batch_system;
98 $queue = "prod" unless defined $queue;
99 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
100 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
101 $time_limit = "60:00:00";
102 $ref_sample_path = 0 unless defined $ref_sample_path;
103 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
104 $xspl_file = "$softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml";
105 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-mctest";
106 $mcseed = 210921029;
107 
108 %nevents_hash = (
109  '1000' => '100000',
110  '1001' => '100000',
111  '1002' => '100000',
112  '1003' => '100000',
113  '1101' => '100000',
114  '1102' => '100000',
115  '1103' => '100000',
116  '2001' => '100000',
117  '2002' => '100000',
118  '2003' => '100000',
119  '2101' => '100000',
120  '2102' => '100000',
121  '2103' => '100000',
122  '9001' => '100000',
123  '9002' => '100000',
124  '9101' => '100000',
125  '9201' => '50000',
126  '9202' => '50000',
127  '9203' => '50000',
128  '9204' => '50000'
129 );
130 
131 %nupdg_hash = (
132  '1000' => '14',
133  '1001' => '14',
134  '1002' => '14',
135  '1003' => '14',
136  '1101' => '-14',
137  '1102' => '-14',
138  '1103' => '-14',
139  '2001' => '14',
140  '2002' => '14',
141  '2003' => '14',
142  '2101' => '-14',
143  '2102' => '-14',
144  '2103' => '-14',
145  '9001' => '14',
146  '9002' => '14',
147  '9101' => '14',
148  '9201' => '12',
149  '9202' => '14',
150  '9203' => '14',
151  '9204' => '-12'
152 );
153 
154 %tgtpdg_hash = (
155  '1000' => '1000000010',
156  '1001' => '1000000010',
157  '1002' => '1000000010',
158  '1003' => '1000000010',
159  '1101' => '1000010010',
160  '1102' => '1000010010',
161  '1103' => '1000010010',
162  '2001' => '1000260560',
163  '2002' => '1000260560',
164  '2003' => '1000260560',
165  '2101' => '1000260560',
166  '2102' => '1000260560',
167  '2103' => '1000260560',
168  '9001' => '1000260560',
169  '9002' => '1000260560',
170  '9101' => '1000260560',
171  '9201' => '1000260560',
172  '9202' => '1000260560',
173  '9203' => '1000260560',
174  '9204' => '1000260560'
175 );
176 
177 %energy_hash = (
178  '1000' => '0.5',
179  '1001' => '1.0',
180  '1002' => '5.0',
181  '1003' => '50.0',
182  '1101' => '1.0',
183  '1102' => '5.0',
184  '1103' => '50.0',
185  '2001' => '1.0',
186  '2002' => '5.0',
187  '2003' => '50.0',
188  '2101' => '1.0',
189  '2102' => '5.0',
190  '2103' => '50.0',
191  '9001' => '5.0',
192  '9002' => '5.0',
193  '9101' => '2.0',
194  '9201' => '1.0',
195  '9202' => '1.0',
196  '9203' => '20.0',
197  '9204' => '20.0'
198 );
199 
200 %gevgl_hash = (
201  '1000' => 'Default',
202  '1001' => 'Default',
203  '1002' => 'Default',
204  '1003' => 'Default',
205  '1101' => 'Default',
206  '1102' => 'Default',
207  '1103' => 'Default',
208  '2001' => 'Default',
209  '2002' => 'Default',
210  '2003' => 'Default',
211  '2101' => 'Default',
212  '2102' => 'Default',
213  '2103' => 'Default',
214  '9001' => 'CharmCCDIS',
215  '9002' => 'CharmCCQE',
216  '9101' => 'COH',
217  '9201' => 'NuEElastic',
218  '9202' => 'NuEElastic',
219  '9203' => 'IMD',
220  '9204' => 'IMD'
221 );
222 
223 # make the jobs directory
224 #
225 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
226 
227 print "Input runs: $runnu \n";
228 
229 for my $curr_runnu (keys %gevgl_hash) {
230  # print "Checking benchmark run: ...... $curr_runnu \n";
231 
232  if($runnu=~m/$curr_runnu/ || $runnu eq "all") {
233  print "** matched -> submitting job \n";
234 
235  #
236  # get runnu-dependent info
237  #
238  $nev = $nevents_hash {$curr_runnu};
239  $nu = $nupdg_hash {$curr_runnu};
240  $tgt = $tgtpdg_hash {$curr_runnu};
241  $en = $energy_hash {$curr_runnu};
242  $gevgl = $gevgl_hash {$curr_runnu};
243 
244  $jobname = "mctest-$curr_runnu";
245  $filename_template = "$jobs_dir/$jobname";
246 # $grep_pipe = "grep -B 20 -A 30 -i \"warn\\|error\\|fatal\"";
247  $grep_pipe = "grep -B 20 -A 10 -i fatal";
248  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
249  $evgen_opt = "-n $nev -e $en -p $nu -t $tgt -r $curr_runnu --seed $mcseed --cross-sections $xspl_file --event-generator-list $gevgl";
250  $evgen_cmd = "gevgen $evgen_opt | $grep_pipe &> $filename_template.evgen.log";
251  $conv_cmd = "gntpc -f gst -i gntp.$curr_runnu.ghep.root | $grep_pipe &> $filename_template.conv.log";
252  $comp_cmd = "gevcomp -f gntp.$curr_runnu.gst.root -r $ref_sample_path/gntp.$curr_runnu.gst.root | $grep_pipe &> $filename_template.comp.log";
253 
254  print "@@ exec: $evgen_cmd \n";
255 
256  #
257  # submit
258  #
259 
260  # PBS case
261  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
262  $batch_script = "$filename_template.pbs";
263  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
264  print PBS "#!/bin/bash \n";
265  print PBS "#PBS -N $jobname \n";
266  print PBS "#PBS -o $filename_template.pbsout.log \n";
267  print PBS "#PBS -e $filename_template.pbserr.log \n";
268  print PBS "source $genie_setup \n";
269  print PBS "cd $jobs_dir \n";
270  print PBS "$evgen_cmd \n";
271  print PBS "$conv_cmd \n";
272  if(-d $ref_sample_path) {
273  print PBS "$comp_cmd \n";
274  }
275  close(PBS);
276  $job_submission_command = "qsub";
277  if($batch_system eq 'HTCondor_PBS') {
278  $job_submission_command = "condor_qsub";
279  }
280  `$job_submission_command -q $queue $batch_script`;
281  } #PBS
282 
283  # LSF case
284  if($batch_system eq 'LSF') {
285  $batch_script = "$filename_template.sh";
286  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
287  print LSF "#!/bin/bash \n";
288  print PBS "#BSUB-j $jobname \n";
289  print LSF "#BSUB-q $queue \n";
290  print LSF "#BSUB-o $filename_template.lsfout.log \n";
291  print LSF "#BSUB-e $filename_template.lsferr.log \n";
292  print LSF "source $genie_setup \n";
293  print LSF "cd $jobs_dir \n";
294  print LSF "$evgen_cmd \n";
295  print LSF "$conv_cmd \n";
296  if(-d $ref_sample_path) {
297  print LSF "$comp_cmd \n";
298  }
299  close(LSF);
300  `bsub < $batch_script`;
301  } #LSF
302 
303  # HTCondor
304  if($batch_system eq 'HTCondor') {
305  $batch_script = "$filename_template.htc";
306  open(HTC, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
307  print HTC "Universe = vanilla \n";
308  print HTC "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
309  print HTC "Arguments = $genie_setup $jobs_dir $evgen_cmd $conv_cmd";
310  if(-d $ref_sample_path) {
311  print HTC " $comp_cmd";
312  }
313  print HTC "\n";
314  print HTC "Log = $filename_template.log \n";
315  print HTC "Output = $filename_template.out \n";
316  print HTC "Error = $filename_template.err \n";
317  print HTC "Request_memory = 2 GB \n";
318  print HTC "Queue \n";
319  close(HTC);
320  `condor_submit $batch_script`;
321  } #HTCondor
322 
323  # slurm case
324  if($batch_system eq 'slurm') {
325  my $time_lim = `sinfo -h -p batch -o %l`;
326  my ($days, $hours, $remainder) = $time_lim =~ /([0]+)-([0-9]+):(.*)/;
327  my $newhours = $days * 24 + $hours;
328  my $new_time_lim = "$newhours:$remainder";
329  $time_limit = $new_time_lim lt $time_limit ? $new_time_lim : $time_limit;
330  $batch_script = "$filename_template.sh";
331  open(SLURM, ">$batch_script") or die("Can not create the SLURM batch script");
332  print SLURM "#!/bin/bash \n";
333  print SLURM "#SBATCH-p $queue \n";
334  print SLURM "#SBATCH-o $filename_template.lsfout.log \n";
335  print SLURM "#SBATCH-e $filename_template.lsferr.log \n";
336  print SLURM "#SBATCH-t $time_limit \n";
337  print SLURM "source $genie_setup \n";
338  print SLURM "cd $jobs_dir \n";
339  print SLURM "$evgen_cmd \n";
340  print SLURM "$conv_cmd \n";
341  if(-d $ref_sample_path) {
342  print SLURM "$comp_cmd \n";
343  }
344  close(SLURM);
345  `sbatch --job-name=$jobname $batch_script`;
346  } #slurm
347 
348  # no batch system, run jobs interactively
349  if($batch_system eq 'none') {
350  system("source $genie_setup; cd $jobs_dir; $evgen_cmd; $conv_cmd");
351  if(-d $ref_sample_path){
352  system("$comp_cmd");
353  }
354  } # interactive mode
355 
356  }
357 }
Generated by   doxygen 1.8.11