submit_neutrino_xsec_validation_mc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #---------------------------------------------------------------------------------------------------------------------
4 # Submit jobs to generate all data needed for testing GENIE against the world data on integrated neutrino x-sections.
5 # The generated data can be fed into GENIE/Comparisons gvld_intg_nu_xsec app.
6 #
7 # Syntax:
8 # perl submit_neutrino_xsec_validation_mc_jobs.pl <options>
9 #
10 # Options:
11 # --version : GENIE version number
12 # [--config-dir] : Config directory, default is $GENIE/config
13 # [--nsubruns] : number of subruns per run, default: 1
14 # [--offset] : subrun offset (for augmenting existing sample), default: 0
15 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
16 # [--production] : production name, default: routine_validation
17 # [--cycle] : cycle in current production, default: 01
18 # [--use-valgrind] : default: off
19 # [--batch-system] : <PBS, LyonPBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: PBS
20 # [--queue] : default: prod. LyonPBS default: P_gdrnu_genie
21 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
22 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/softw/scratch/GENIE/
23 # [--spline-file] : absoluyte path to xsec_spline_file, default: $softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml
24 #
25 # EVENT SAMPLES:
26 # The following samples will be submitted.
27 # The output GHEP event trees will be analyzed and converted to GST summary trees.
28 # The validation apps use the samples to decompose the inclusive cross section
29 # to cross section for various exclusive channels
30 #......................................................................
31 # run number | init state | energy | processes
32 # | | (GeV) | enabled
33 #......................................................................
34 #
35 # 1000xx | numu + n | 0.1-120. | all
36 # 1100xx | numu + p | 0.1-120. | all
37 # 1200xx | numubar + n | 0.1-120. | all
38 # 1300xx | numubar + p | 0.1-120. | all
39 # 2010xx | numu + 12C | 0.1-120. | all
40 #
41 # xx : Run ID, 01-99, 100k events each
42 #
43 # CROSS SECTIONS:
44 # A job will be submitted to run GENIE's gspl2root utility, extract
45 # cross sections (from the pre-computed XML cross section spline file)
46 # and save them into a single ROOT file (xsec.root).
47 # The file will contain the following folders (TDirectories):
48 # - nu_mu_n
49 # - nu_mu_H1
50 # - nu_mu_bar_n
51 # - nu_mu_bar_H1
52 # - nu_mu_C12
53 # - nu_mu_bar_C12
54 # - nu_mu_Ne20
55 # - nu_mu_bar_Ne20
56 # - nu_mu_Al27
57 # - nu_mu_Si30 (Si30: proxy for freon (average A=30), used for coherent pi production plots only)
58 # - nu_mu_bar_Si30 ( -//- )
59 # Each folder will contain graphs for all cross sections for the given initial state.
60 # Expand as required if more published data are inluded in this GENIE physics benchmark test.
61 #
62 #
63 # Author:
64 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
65 # University of Liverpool & STFC Rutherford Appleton Laboratory
66 #
67 # Copyright:
68 # Copyright (c) 2003-2020, The GENIE Collaboration
69 # For the full text of the license visit http://copyright.genie-mc.org
70 #---------------------------------------------------------------------------------------------------------------------
71 #
72 
73 use File::Path;
74 
75 # inputs
76 #
77 $iarg=0;
78 foreach (@ARGV) {
79  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
80  if($_ eq '--config-dir') { $config_dir = $ARGV[$iarg+1]; }
81  if($_ eq '--nsubruns') { $nsubruns = $ARGV[$iarg+1]; }
82  if($_ eq '--offset') { $offset = $ARGV[$iarg+1]; }
83  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
84  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
85  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
86  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
87  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
88  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
89  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
90  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
91  if($_ eq '--spline-file') { $xspl_file = $ARGV[$iarg+1]; }
92  $iarg++;
93 }
94 die("** Aborting [Undefined GENIE version. Use the --version option]")
95 unless defined $genie_version;
96 
97 $config_dir = "" unless defined $config_dir;
98 $nsubruns = 1 unless defined $nsubruns;
99 $offset = 0 unless defined $offset;
100 $use_valgrind = 0 unless defined $use_valgrind;
101 $arch = "SL6.x86_64" unless defined $arch;
102 $production = "routine_validation" unless defined $production;
103 $cycle = "01" unless defined $cycle;
104 $batch_system = "PBS" unless defined $batch_system;
105 $queue_default = "prod" ;
106 if ( $batch_system eq 'LyonPBS' ) {
107  $queue_default = "P_gdrnu_genie" ;
108 }
109 $queue = $queue_default unless defined $queue;
110 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
111 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
112 $xspl_file = "$softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml" unless defined $xspl_file ;
113 $time_limit = "60:00:00";
114 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup $config_dir";
115 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-xsec_validation";
116 $mcseed = 210921029;
117 $nev_per_subrun = 100000;
118 
119 # inputs for event generation jobs
120 %evg_nupdg_hash = (
121  '1000' => '14',
122  '1100' => '14',
123  '1200' => '-14',
124  '1300' => '-14',
125  '2010' => '14'
126 );
127 %evg_tgtpdg_hash = (
128  '1000' => '1000000010',
129  '1100' => '1000010010',
130  '1200' => '1000000010',
131  '1300' => '1000010010',
132  '2010' => '1000060120'
133 );
134 
135 %evg_energy_hash = (
136  '1000' => '0.1,120.0',
137  '1100' => '0.1,120.0',
138  '1200' => '0.1,120.0',
139  '1300' => '0.1,120.0',
140  '2010' => '0.1,120.0'
141 );
142 %evg_gevgl_hash = (
143  '1000' => 'Default',
144  '1100' => 'Default',
145  '1200' => 'Default',
146  '1300' => 'Default',
147  '2010' => 'Default'
148 );
149 %evg_fluxopt_hash = (
150  '1000' => '-f "1/x"',
151  '1100' => '-f "1/x"',
152  '1200' => '-f "1/x"',
153  '1300' => '-f "1/x"',
154  '2010' => '-f "1/x"'
155 );
156 
157 # inputs for xsec calculation jobs
158 %xsec_nupdg_hash = (
159  '1' => '14',
160  '2' => '14',
161  '3' => '-14',
162  '4' => '-14',
163  '5' => '14',
164  '6' => '-14',
165  '7' => '14',
166  '8' => '14',
167  '9' => '-14',
168  '10' => '14',
169  '11' => '-14'
170 );
171 %xsec_tgtpdg_hash = (
172  '1' => '1000000010',
173  '2' => '1000010010',
174  '3' => '1000000010',
175  '4' => '1000010010',
176  '5' => '1000100200', # Ne20
177  '6' => '1000100200', # Ne20
178  '7' => '1000130270', # Al27
179  '8' => '1000140300', # Si30, proxy for freon (average A=30), used for coherent pi production plots only
180  '9' => '1000140300', # Si30, proxy for freon (average A=30), used for coherent pi production plots only
181  '10' => '1000060120', # C12
182  '11' => '1000060120' # C12
183 );
184 
185 # make the jobs directory
186 #
187 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
188 
189 #
190 # submit event generation jobs
191 #
192 for my $curr_runnu (keys %evg_gevgl_hash) {
193 
194  # uncomment if you want to include a cmd line input to specify specific runs (and fill-in $runnu)
195  # if($runnu=~m/$curr_runnu/ || $runnu eq "all") {
196 
197  print "** submitting event generation run: $curr_runnu \n";
198 
199  #
200  # get runnu-dependent info
201  #
202  $nu = $evg_nupdg_hash {$curr_runnu};
203  $tgt = $evg_tgtpdg_hash {$curr_runnu};
204  $en = $evg_energy_hash {$curr_runnu};
205  $gevgl = $evg_gevgl_hash {$curr_runnu};
206  $fluxopt = $evg_fluxopt_hash {$curr_runnu};
207 
208  # submit subruns
209  for($isubrun = 0; $isubrun < $nsubruns; $isubrun++) {
210 
211  $curr_subrunnu = 100 * $curr_runnu + $isubrun + $offset;
212  $jobname = "xsecvld-$curr_subrunnu";
213  $filename_template = "$jobs_dir/$jobname";
214  $curr_seed = $mcseed + $isubrun + $offset;
215  $grep_pipe = "grep -B 100 -A 30 -i \"warn\\|error\\|fatal\"";
216  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
217  $evgen_opt = "-n $nev_per_subrun -e $en -p $nu -t $tgt $fluxopt -r $curr_subrunnu --seed $curr_seed --cross-sections $xspl_file --event-generator-list $gevgl";
218  $evgen_cmd = "gevgen $evgen_opt | $grep_pipe &> $filename_template.evgen.log";
219  $conv_cmd = "gntpc -f gst -i gntp.$curr_subrunnu.ghep.root | $grep_pipe &> $filename_template.conv.log";
220 
221  print "@@ exec: $evgen_cmd \n";
222 
223  #
224  # submit
225  #
226 
227  # PBS case
228  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
229  $batch_script = "$filename_template.pbs";
230  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
231  print PBS "#!/bin/bash \n";
232  print PBS "#PBS -N $jobname \n";
233  print PBS "#PBS -l cput=$time_limit \n";
234  print PBS "#PBS -o $filename_template.pbsout.log \n";
235  print PBS "#PBS -e $filename_template.pbserr.log \n";
236  print PBS "source $genie_setup \n";
237  print PBS "cd $jobs_dir \n";
238  print PBS "$evgen_cmd \n";
239  print PBS "$conv_cmd \n";
240  close(PBS);
241  $job_submission_command = "qsub";
242  if($batch_system eq 'HTCondor_PBS') {
243  $job_submission_command = "condor_qsub";
244  }
245  `$job_submission_command -q $queue $batch_script`;
246  } #PBS
247 
248  # LyonPBS case
249  if($batch_system eq 'LyonPBS' ) {
250  $batch_script = "$filename_template.pbs";
251  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
252  print PBS "#!/bin/bash \n";
253  print PBS "#\$ -P $queue \n";
254  print PBS "#\$ -N $jobname \n";
255  print PBS "#\$ -o $filename_template.pbsout.log \n";
256  print PBS "#\$ -e $filename_template.pbserr.log \n";
257  print PBS "#\$ -l ct=6:00:00,sps=1 \n";
258  print PBS "source $genie_setup \n";
259  print PBS "cd $jobs_dir \n";
260  print PBS "$evgen_cmd \n";
261  print PBS "$conv_cmd \n";
262  close(PBS);
263  $job_submission_command = "qsub";
264  `$job_submission_command $batch_script`;
265  } #LyonPBS
266 
267 
268  # LSF case
269  if($batch_system eq 'LSF') {
270  $batch_script = "$filename_template.sh";
271  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
272  print LSF "#!/bin/bash \n";
273  print LSF "#BSUB-j $jobname \n";
274  print LSF "#BSUB-q $queue \n";
275  print LSF "#BSUB-c $time_limit \n";
276  print LSF "#BSUB-o $filename_template.lsfout.log \n";
277  print LSF "#BSUB-e $filename_template.lsferr.log \n";
278  print LSF "source $genie_setup \n";
279  print LSF "cd $jobs_dir \n";
280  print LSF "$evgen_cmd \n";
281  print LSF "$conv_cmd \n";
282  close(LSF);
283  `bsub < $batch_script`;
284  } #LSF
285 
286  # HTCondor
287  if($batch_system eq 'HTCondor') {
288  $batch_script = "$filename_template.htc";
289  open(HTC, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
290  print HTC "Universe = vanilla \n";
291  print HTC "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
292  print HTC "Arguments = $genie_setup $jobs_dir $evgen_cmd $conv_cmd\n";
293  print HTC "Log = $filename_template.log \n";
294  print HTC "Output = $filename_template.out \n";
295  print HTC "Error = $filename_template.err \n";
296  print HTC "Request_memory = 2 GB \n";
297  print HTC "Queue \n";
298  close(HTC);
299  `condor_submit $batch_script`;
300  } #HTCondor
301 
302  # slurm case
303  if($batch_system eq 'slurm') {
304  my $time_lim = `sinfo -h -p batch -o %l`;
305  my ($days, $hours, $remainder) = $time_lim =~ /([0]+)-([0-9]+):(.*)/;
306  my $newhours = $days * 24 + $hours;
307  my $new_time_lim = "$newhours:$remainder";
308  $time_limit = $new_time_lim lt $time_limit ? $new_time_lim : $time_limit;
309  $batch_script = "$filename_template.sh";
310  open(SLURM, ">$batch_script") or die("Can not create the SLURM batch script");
311  print SLURM "#!/bin/bash \n";
312  print SLURM "#SBATCH-p $queue \n";
313  print SLURM "#SBATCH-o $filename_template.lsfout.log \n";
314  print SLURM "#SBATCH-e $filename_template.lsferr.log \n";
315  print SLURM "#SBATCH-t $time_limit \n";
316  print SLURM "source $genie_setup \n";
317  print SLURM "cd $jobs_dir \n";
318  print SLURM "$evgen_cmd \n";
319  print SLURM "$conv_cmd \n";
320  close(SLURM);
321  `sbatch --job-name=$jobname $batch_script`;
322  } #slurm
323 
324  # no batch system, run jobs interactively
325  if($batch_system eq 'none') {
326  system("source $genie_setup; cd $jobs_dir; $evgen_cmd; $conv_cmd");
327  } # interactive mode
328 
329  } # loop over subruns
330  # } #checking whether to submit current run
331 } # loop over runs
332 
333 
334 #
335 # submit job to generate the cross section ROOT file
336 #
337 {
338  $gspl2root_cmd = "";
339  for my $curr_init_state (keys %xsec_nupdg_hash) {
340  $nu = $xsec_nupdg_hash {$curr_init_state};
341  $tgt = $xsec_tgtpdg_hash {$curr_init_state};
342  $gspl2root_cmd = "$gspl2root_cmd \ngspl2root -p $nu -t $tgt -f $xspl_file -o xsec.root";
343  }
344  print "@@@ exec: $gspl2root_cmd \n";
345 
346  $filename_template = "$jobs_dir/xsecvld-xsec";
347 
348  # PBS case
349  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
350  $batch_script = "$filename_template.pbs";
351  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
352  print PBS "#!/bin/bash \n";
353  print PBS "#PBS -l cput=$time_limit \n";
354  print PBS "#PBS -o $filename_template.pbsout.log \n";
355  print PBS "#PBS -e $filename_template.pbserr.log \n";
356  print PBS "source $genie_setup \n";
357  print PBS "cd $jobs_dir \n";
358  print PBS "$gspl2root_cmd \n";
359  close(PBS);
360  $job_submission_command = "qsub";
361  if($batch_system eq 'HTCondor_PBS') {
362  $job_submission_command = "condor_qsub";
363  }
364  `$job_submission_command -q $queue $batch_script`;
365  } #PBS
366 
367  # LyonPBS case
368  if($batch_system eq 'LyonPBS' ) {
369  $batch_script = "$filename_template.pbs";
370  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
371  print PBS "#!/bin/bash \n";
372  print PBS "#\$ -P $queue \n";
373  print PBS "#\$ -N $jobname \n";
374  print PBS "#\$ -o $filename_template.pbsout.log \n";
375  print PBS "#\$ -e $filename_template.pbserr.log \n";
376  print PBS "#\$ -l ct=6:00:00,sps=1 \n";
377  print PBS "source $genie_setup \n";
378  print PBS "cd $jobs_dir \n";
379  print PBS "$gspl2root_cmd \n";
380  close(PBS);
381  $job_submission_command = "qsub";
382  `$job_submission_command $batch_script`;
383  } #LyonPBS
384 
385 
386  # LSF case
387  if($batch_system eq 'LSF') {
388  $batch_script = "$filename_template.sh";
389  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
390  print LSF "#!/bin/bash \n";
391  print LSF "#BSUB-q $queue \n";
392  print LSF "#BSUB-c $time_limit \n";
393  print LSF "#BSUB-o $filename_template.lsfout.log \n";
394  print LSF "#BSUB-e $filename_template.lsferr.log \n";
395  print LSF "source $genie_setup \n";
396  print LSF "cd $jobs_dir \n";
397  print LSF "$gspl2root_cmd \n";
398  close(LSF);
399  `bsub < $batch_script`;
400  } #LSF
401 
402  # HTCondor
403  if($batch_system eq 'HTCondor') {
404  $batch_script = "$filename_template.htc";
405  open(HTC, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
406  print HTC "Universe = vanilla \n";
407  print HTC "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
408  print HTC "Arguments = $genie_setup $jobs_dir $gspl2root_cmd \n";
409  print HTC "Log = $filename_template.log \n";
410  print HTC "Output = $filename_template.out \n";
411  print HTC "Error = $filename_template.err \n";
412  print HTC "Request_memory = 2 GB \n";
413  print HTC "Queue \n";
414  close(HTC);
415  `condor_submit $batch_script`;
416  } #HTCondor
417 
418  # slurm case
419  if($batch_system eq 'slurm') {
420  $batch_script = "$filename_template.sh";
421  open(SLURM, ">$batch_script") or die("Can not create the SLURM batch script");
422  print SLURM "#!/bin/bash \n";
423  print SLURM "#SBATCH-p $queue \n";
424  print SLURM "#SBATCH-o $filename_template.lsfout.log \n";
425  print SLURM "#SBATCH-e $filename_template.lsferr.log \n";
426  print SLURM "source $genie_setup \n";
427  print SLURM "cd $jobs_dir \n";
428  print SLURM "$gspl2root_cmd \n";
429  close(SLURM);
430  `sbatch $batch_script`;
431  } #slurm
432 
433  # no batch system, run jobs interactively
434  if($batch_system eq 'none') {
435  system("source $genie_setup; cd $jobs_dir; $gsp12root_cmd");
436  } # interactive mode
437 
438 }
Generated by   doxygen 1.8.11