submit_t2k_validation_mc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #---------------------------------------------------------------------------------------------------------------------
4 # Submit jobs for validating GENIE against the T2K data releases.
5 # The outputs can be fed to appropriate apps of the GENIE/Comparisons product
6 #
7 # Syntax:
8 # shell% perl submit_t2k_validation_mc_jobs.pl <options>
9 #
10 # Options:
11 # --version : GENIE version number
12 # --run-type : Comma separated list of run types
13 # [--config-dir] : Config directory, default is $GENIE/config
14 # [--nsubruns] : number of subruns per run, default: 1
15 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
16 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
17 # [--cycle] : default: 01
18 # [--batch-system] : <PBS, LyonPBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: HTCondor_PBS
19 # [--queue] : default: prod. LyonPBS default: P_gdrnu_genie
20 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
21 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
22 # [--spline-file] : absoluyte path to xsec_spline_file, default: $softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml
23 #
24 #
25 # SAMPLES:
26 #.............................................................................................
27 # run | nev | init state | location | flux | relevant data releases
28 # type | /subrun | | | | in GENIE/Comparisons
29 #........................................................................
30 # 100 | 100k | numu + Carbon | on-axis | (a) | t2k_ingrid_numuccqe_2015
31 # 110 | 100k | numu + Carbon | off-axis | (b) | t2k_nd280_numuccqe_2014, t2k_nd280_numucc_2013
32 # 210 | 100k | nue + Carbon | off-axis | (c) | t2k_nd280_nuecc_2014
33 #........................................................................
34 #
35 # Run numbers are constructed as IJKXXX, where IJK is the run type and XXX is the subrun number
36 #
37 # I: Flavour (1: numu, 2: nue)
38 # J: Location (0: on-axis/INGRID, 1: off-axis/ND280)
39 # K: Nuclear target (0: Carbon)
40 #
41 # Info on flux used:
42 # (a) see the code for now
43 # (b) see the code for now
44 # (c) see the code for now
45 #
46 # Author:
47 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
48 # University of Liverpool & STFC Rutherford Appleton Laboratory
49 #
50 # Copyright:
51 # Copyright (c) 2003-2020, The GENIE Collaboration
52 # For the full text of the license visit http://copyright.genie-mc.org
53 #---------------------------------------------------------------------------------------------------------------------
54 #
55 #
56 
57 use File::Path;
58 
59 # inputs
60 #
61 $iarg=0;
62 foreach (@ARGV) {
63  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
64  if($_ eq '--run-type') { $run_type = $ARGV[$iarg+1]; }
65  if($_ eq '--config-dir') { $config_dir = $ARGV[$iarg+1]; }
66  if($_ eq '--nsubruns') { $nsubruns = $ARGV[$iarg+1]; }
67  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
68  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
69  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
70  if($_ eq '--ref-samples') { $ref_sample_path = $ARGV[$iarg+1]; }
71  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
72  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
73  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
74  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
75  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
76  if($_ eq '--spline-file') { $xspl_file = $ARGV[$iarg+1]; }
77  $iarg++;
78 }
79 die("** Aborting [Undefined benchmark runs #. Use the --run-type option]")
80 unless defined $run_type;
81 die("** Aborting [Undefined GENIE version. Use the --version option]")
82 unless defined $genie_version;
83 
84 $config_dir = "" unless defined $config_dir;
85 $nsubruns = 1 unless defined $nsubruns;
86 $use_valgrind = 0 unless defined $use_valgrind;
87 $arch = "SL6.x86_64" unless defined $arch;
88 $production = "routine_validation" unless defined $production;
89 $cycle = "01" unless defined $cycle;
90 $batch_system = "PBS" unless defined $batch_system;
91 $queue_default = "prod" ;
92 if ( $batch_system eq 'LyonPBS' ) {
93  $queue_default = "P_gdrnu_genie" ;
94 }
95 $queue = $queue_default unless defined $queue;
96 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
97 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
98 $ref_sample_path = 0 unless defined $ref_sample_path;
99 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
100 $xspl_file = "$softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml" unless defined $xspl_file;
101 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-t2k";
102 $mcseed = 210921029;
103 
104 %nevents_hash = (
105  '100' => '100000',
106  '110' => '100000',
107  '210' => '100000'
108 );
109 
110 %nupdg_hash = (
111  '100' => '14',
112  '110' => '14',
113  '210' => '12',
114 );
115 
116 %tgtpdg_hash = (
117  '100' => '1000060120',
118  '110' => '1000060120',
119  '210' => '1000060120'
120 );
121 
122 %energy_hash = (
123  '100' => '0,20',
124  '110' => '0,20',
125  '210' => '0,20'
126 );
127 
128 %flux_hash = (
129  '100' => "$softw_topdir/comparisons/builds/$arch/vtrunk/data/measurements/vA/t2k/t2k_ingrid_numuccqe_2015/flux_data_release.root,flux",
130  '110' => "$softw_topdir/comparisons/builds/$arch/vtrunk/data/measurements/vA/t2k/t2k_nd280_numucc_2013/data_release.root,flux_numu",
131  '210' => "$softw_topdir/comparisons/builds/$arch/vtrunk/data/measurements/vA/t2k/t2k_nd280_nuecc_2014/flux.root,nue_flux"
132 );
133 
134 # make the jobs directory
135 #
136 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
137 
138 print "Input runs: $run_type \n";
139 
140 for my $curr_runtype (keys %nupdg_hash) {
141  print "Checking benchmark run: ...... $curr_runtype \n";
142 
143  if($run_type=~m/$curr_runtype/ || $run_type eq "all") {
144  print "** matched -> submitting job \n";
145 
146  # get run type dependent info
147  $nev = $nevents_hash {$curr_runtype};
148  $nu = $nupdg_hash {$curr_runtype};
149  $tgt = $tgtpdg_hash {$curr_runtype};
150  $en = $energy_hash {$curr_runtype};
151  $flux = $flux_hash {$curr_runtype};
152 
153  # submit subruns
154  for($subrunnu = 0; $subrunnu < $nsubruns; $subrunnu++) {
155 
156  $runnu = 1000 * $curr_runtype + $subrunnu;
157  $jobname = "t2k-$runnu";
158  $filename_template = "$jobs_dir/$jobname";
159 # $grep_pipe = "grep -B 20 -A 30 -i \"warn\\|error\\|fatal\"";
160  $grep_pipe = "grep -B 20 -A 30 -i fatal";
161  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
162  $curr_seed = $mcseed + $runnu;
163  $evgen_opt = "-n $nev -p $nu -t $tgt -r $runnu --seed $curr_seed --cross-sections $xspl_file -e $en -f $flux";
164  $evgen_cmd = "gevgen $evgen_opt | $grep_pipe &> $filename_template.evgen.log";
165 
166  print "@@ exec: $evgen_cmd \n";
167 
168  #
169  # submit
170  #
171 
172  # PBS case
173  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
174  $batch_script = "$filename_template.pbs";
175  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the PBS batch script");
176  print BATCH_SCRIPT "#!/bin/bash \n";
177  print BATCH_SCRIPT "#PBS -N $jobname \n";
178  print BATCH_SCRIPT "#PBS -o $filename_template.pbsout.log \n";
179  print BATCH_SCRIPT "#PBS -e $filename_template.pbserr.log \n";
180  print BATCH_SCRIPT "source $genie_setup $config_dir \n";
181  print BATCH_SCRIPT "cd $jobs_dir \n";
182  print BATCH_SCRIPT "$evgen_cmd \n";
183  close(BATCH_SCRIPT);
184  $job_submission_command = "qsub";
185  if($batch_system eq 'HTCondor_PBS') {
186  $job_submission_command = "condor_qsub";
187  }
188  `$job_submission_command -q $queue $batch_script`;
189  } #PBS
190 
191  # LyonPBS case
192  if($batch_system eq 'LyonPBS' ) {
193  $batch_script = "$filename_template.pbs";
194  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
195  print PBS "#!/bin/bash \n";
196  print PBS "#\$ -P $queue \n";
197  print PBS "#\$ -N $jobname \n";
198  print PBS "#\$ -o $filename_template.pbsout.log \n";
199  print PBS "#\$ -e $filename_template.pbserr.log \n";
200  print PBS "#\$ -l ct=6:00:00,sps=1 \n";
201  print PBS "source $genie_setup $config_dir \n";
202  print PBS "cd $jobs_dir \n";
203  print PBS "$evgen_cmd \n";
204  close(PBS);
205  $job_submission_command = "qsub";
206  `$job_submission_command $batch_script`;
207  } #LyonPBS
208 
209  # LSF case
210  if($batch_system eq 'LSF') {
211  $batch_script = "$filename_template.sh";
212  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the LSF batch script");
213  print BATCH_SCRIPT "#!/bin/bash \n";
214  print BATCH_SCRIPT "#BSUB-j $jobname \n";
215  print BATCH_SCRIPT "#BSUB-q $queue \n";
216  print BATCH_SCRIPT "#BSUB-o $filename_template.lsfout.log \n";
217  print BATCH_SCRIPT "#BSUB-e $filename_template.lsferr.log \n";
218  print BATCH_SCRIPT "source $genie_setup $config_dir\n";
219  print BATCH_SCRIPT "cd $jobs_dir \n";
220  print BATCH_SCRIPT "$evgen_cmd \n";
221  close(BATCH_SCRIPT);
222  `bsub < $batch_script`;
223  } #LSF
224 
225  # HTCondor
226  if($batch_system eq 'HTCondor') {
227  $batch_script = "$filename_template.htc";
228  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
229  print BATCH_SCRIPT "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
230  print BATCH_SCRIPT "Arguments = $genie_setup $jobs_dir $gevgen_cmd \n";
231  print BATCH_SCRIPT "Log = $filename_template.log \n";
232  print BATCH_SCRIPT "Output = $filename_template.out \n";
233  print BATCH_SCRIPT "Error = $filename_template.err \n";
234  print BATCH_SCRIPT "Universe = vanilla \n";
235  print BATCH_SCRIPT "Request_memory = 2 GB \n";
236  print BATCH_SCRIPT "Getenv = True \n";
237  print BATCH_SCRIPT "should_transfer_files = YES \n";
238  print BATCH_SCRIPT "when_to_transfer_output = ON_EXIT \n";
239  print BATCH_SCRIPT "\nqueue\n";
240  print BATCH_SCRIPT "Queue \n";
241  close(BATCH_SCRIPT);
242  `condor_submit $batch_script`;
243  } #HTCondor
244 
245  # slurm case
246  if($batch_system eq 'slurm') {
247  $batch_script = "$filename_template.sh";
248  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the SLURM batch script");
249  print BATCH_SCRIPT "#!/bin/bash \n";
250  print BATCH_SCRIPT "#SBATCH-p $queue \n";
251  print BATCH_SCRIPT "#SBATCH-o $filename_template.lsfout.log \n";
252  print BATCH_SCRIPT "#SBATCH-e $filename_template.lsferr.log \n";
253  print BATCH_SCRIPT "source $genie_setup $config_dir\n";
254  print BATCH_SCRIPT "cd $jobs_dir \n";
255  print BATCH_SCRIPT "$evgen_cmd \n";
256  close(BATCH_SCRIPT);
257  `sbatch --job-name=$jobname $batch_script`;
258  } #slurm
259 
260  # no batch system, run jobs interactively
261  if($batch_system eq 'none') {
262  system("source $genie_setup; cd $jobs_dir $config_dir; $evgen_cmd");
263  } # interactive mode
264 
265  }#subrunnu
266 
267  }
268 }
Generated by   doxygen 1.8.11