submit_miniboone_validation_mc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #---------------------------------------------------------------------------------------------------------------------
4 # Submit jobs for validating GENIE against the MiniBooNE data releases.
5 # The outputs can be fed to appropriate apps of the GENIE/Comparisons product
6 #
7 # Syntax:
8 # shell% perl submit_miniboone_validation_mc_jobs.pl <options>
9 #
10 # Options:
11 # --version : GENIE version number
12 # --run-type : Comma separated list of run types
13 # [--config-dir] : Config directory, default is $GENIE/config
14 # [--nsubruns] : number of subruns per run, default: 1
15 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
16 # [--production] : default: routine_validation
17 # [--cycle] : default: 01
18 # [--batch-system] : <PBS, LyonPBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: HTCondor_PBS
19 # [--queue] : default: prod. LyonPBS default: P_gdrnu_genie
20 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
21 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
22 # [--spline-file] : absoluyte path to xsec_spline_file, default: $softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml
23 #
24 #
25 # SAMPLES:
26 #........................................................................
27 # run | nev | init state | energy | flux
28 # type | /subrun | | |
29 #........................................................................
30 # 10 | 100k | numu + Hydrocarbon | 0 - 10 GeV | (a)
31 # 20 | 100k | numubar + Hydrocarbon | 0 - 10 GeV | (b)
32 #........................................................................
33 #
34 # Run numbers are constructed as TTXXX, where TT is the run type and XXX is the subrun number
35 #
36 # Hydrocarbon: 85.7142% C12 + 14.2857% H1
37 # (a) april07_baseline_rgen610.6_flux_pospolarity_fluxes (flux_pos_pol_numu)
38 # (b) december2007_horn-174ka_rgen610.6_flux_negpolarity_fluxes (flux_neg_pol_numub)
39 #
40 # Author:
41 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
42 # University of Liverpool & STFC Rutherford Appleton Laboratory
43 #
44 # Copyright:
45 # Copyright (c) 2003-2020, The GENIE Collaboration
46 # For the full text of the license visit http://copyright.genie-mc.org
47 #---------------------------------------------------------------------------------------------------------------------
48 #
49 #
50 
51 use File::Path;
52 
53 # inputs
54 #
55 $iarg=0;
56 foreach (@ARGV) {
57  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
58  if($_ eq '--config-dir') { $config_dir = $ARGV[$iarg+1]; }
59  if($_ eq '--run-type') { $run_type = $ARGV[$iarg+1]; }
60  if($_ eq '--nsubruns') { $nsubruns = $ARGV[$iarg+1]; }
61  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
62  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
63  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
64  if($_ eq '--ref-samples') { $ref_sample_path = $ARGV[$iarg+1]; }
65  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
66  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
67  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
68  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
69  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
70  if($_ eq '--spline-file') { $xspl_file = $ARGV[$iarg+1]; }
71  $iarg++;
72 }
73 die("** Aborting [Undefined benchmark runs #. Use the --run-type option]")
74 unless defined $run_type;
75 die("** Aborting [Undefined GENIE version. Use the --version option]")
76 unless defined $genie_version;
77 
78 $config_dir = "" unless defined $config_dir;
79 $nsubruns = 1 unless defined $nsubruns;
80 $use_valgrind = 0 unless defined $use_valgrind;
81 $arch = "SL6.x86_64" unless defined $arch;
82 $production = "routine_validation" unless defined $production;
83 $cycle = "01" unless defined $cycle;
84 $batch_system = "HTCondor_PBS" unless defined $batch_system;
85 $queue_default = "prod" ;
86 if ( $batch_system eq 'LyonPBS' ) {
87  $queue_default = "P_gdrnu_genie" ;
88 }
89 $queue = $queue_default unless defined $queue;
90 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
91 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
92 $ref_sample_path = 0 unless defined $ref_sample_path;
93 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
94 $xspl_file = "$softw_topdir/data/job_inputs/xspl/gxspl-vA-$genie_version.xml" unless defined $xspl_file ;
95 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-miniboone";
96 $mcseed = 210921029;
97 
98 %nevents_hash = (
99  '10' => '100000',
100  '20' => '100000'
101 );
102 
103 %nupdg_hash = (
104  '10' => '14',
105  '20' => '-14'
106 );
107 
108 %tgtpdg_hash = (
109  '10' => '1000060120[0.857142],1000010010[0.142857]',
110  '20' => '1000060120[0.857142],1000010010[0.142857]'
111 );
112 
113 %energy_hash = (
114  '10' => '0,10',
115  '20' => '0,10'
116 );
117 
118 %flux_hash = (
119  '10' => "$softw_topdir/comparisons/builds/$arch/vtrunk/data/fluxes/miniboone/miniboone_april07_baseline_rgen610.6_flux_pospolarity_fluxes.root,flux_pos_pol_numu",
120  '20' => "$softw_topdir/comparisons/builds/$arch/vtrunk/data/fluxes/miniboone/miniboone_december2007_horn-174ka_rgen610.6_flux_negpolarity_fluxes.root,flux_neg_pol_numub"
121 );
122 
123 # make the jobs directory
124 #
125 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
126 
127 print "Input runs: $run_type \n";
128 
129 for my $curr_runtype (keys %nupdg_hash) {
130  print "Checking benchmark run: ...... $curr_runtype \n";
131 
132  if($run_type=~m/$curr_runtype/ || $run_type eq "all") {
133  print "** matched -> submitting job \n";
134 
135  # get run type dependent info
136  $nev = $nevents_hash {$curr_runtype};
137  $nu = $nupdg_hash {$curr_runtype};
138  $tgt = $tgtpdg_hash {$curr_runtype};
139  $en = $energy_hash {$curr_runtype};
140  $flux = $flux_hash {$curr_runtype};
141 
142  # submit subruns
143  for($subrunnu = 0; $subrunnu < $nsubruns; $subrunnu++) {
144 
145  $runnu = 1000 * $curr_runtype + $subrunnu;
146  $jobname = "miniboone-$runnu";
147  $filename_template = "$jobs_dir/$jobname";
148 # $grep_pipe = "grep -B 20 -A 30 -i \"warn\\|error\\|fatal\"";
149  $grep_pipe = "grep -B 20 -A 30 -i fatal";
150  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
151  $curr_seed = $mcseed + $runnu;
152  $evgen_opt = "-n $nev -p $nu -t $tgt -r $runnu --seed $curr_seed --cross-sections $xspl_file -e $en -f $flux";
153  $evgen_cmd = "gevgen $evgen_opt | $grep_pipe &> $filename_template.evgen.log";
154 
155  print "@@ exec: $evgen_cmd \n";
156 
157  #
158  # submit
159  #
160 
161  # PBS case
162  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
163  $batch_script = "$filename_template.pbs";
164  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the PBS batch script");
165  print BATCH_SCRIPT "#!/bin/bash \n";
166  print BATCH_SCRIPT "#PBS -N $jobname \n";
167  print BATCH_SCRIPT "#PBS -o $filename_template.pbsout.log \n";
168  print BATCH_SCRIPT "#PBS -e $filename_template.pbserr.log \n";
169  print BATCH_SCRIPT "source $genie_setup $config_dir \n";
170  print BATCH_SCRIPT "cd $jobs_dir \n";
171  print BATCH_SCRIPT "$evgen_cmd \n";
172  close(BATCH_SCRIPT);
173  $job_submission_command = "qsub";
174  if($batch_system eq 'HTCondor_PBS') {
175  $job_submission_command = "condor_qsub";
176  }
177  `$job_submission_command -q $queue $batch_script`;
178  } #PBS
179 
180  # LyonPBS case
181  if($batch_system eq 'LyonPBS' ) {
182  $batch_script = "$filename_template.pbs";
183  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
184  print PBS "#!/bin/bash \n";
185  print PBS "#\$ -P $queue \n";
186  print PBS "#\$ -N $jobname \n";
187  print PBS "#\$ -o $filename_template.pbsout.log \n";
188  print PBS "#\$ -e $filename_template.pbserr.log \n";
189  print PBS "#\$ -l ct=6:00:00,sps=1 \n";
190  print PBS "source $genie_setup $config_dir \n";
191  print PBS "cd $jobs_dir \n";
192  print PBS "$evgen_cmd \n";
193  close(PBS);
194  $job_submission_command = "qsub";
195  `$job_submission_command $batch_script`;
196  } #LyonPBS
197 
198  # LSF case
199  if($batch_system eq 'LSF') {
200  $batch_script = "$filename_template.sh";
201  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the LSF batch script");
202  print BATCH_SCRIPT "#!/bin/bash \n";
203  print BATCH_SCRIPT "#BSUB-j $jobname \n";
204  print BATCH_SCRIPT "#BSUB-q $queue \n";
205  print BATCH_SCRIPT "#BSUB-o $filename_template.lsfout.log \n";
206  print BATCH_SCRIPT "#BSUB-e $filename_template.lsferr.log \n";
207  print BATCH_SCRIPT "source $genie_setup $config_dir\n";
208  print BATCH_SCRIPT "cd $jobs_dir \n";
209  print BATCH_SCRIPT "$evgen_cmd \n";
210  close(BATCH_SCRIPT);
211  `bsub < $batch_script`;
212  } #LSF
213 
214  # HTCondor
215  if($batch_system eq 'HTCondor') {
216  $batch_script = "$filename_template.htc";
217  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
218  print BATCH_SCRIPT "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
219  print BATCH_SCRIPT "Arguments = $genie_setup $jobs_dir $gevgen_cmd \n";
220  print BATCH_SCRIPT "Log = $filename_template.log \n";
221  print BATCH_SCRIPT "Output = $filename_template.out \n";
222  print BATCH_SCRIPT "Error = $filename_template.err \n";
223  print BATCH_SCRIPT "Universe = vanilla \n";
224  print BATCH_SCRIPT "Request_memory = 2 GB \n";
225  print BATCH_SCRIPT "Getenv = True \n";
226  print BATCH_SCRIPT "should_transfer_files = YES \n";
227  print BATCH_SCRIPT "when_to_transfer_output = ON_EXIT \n";
228  print BATCH_SCRIPT "\nqueue\n";
229  print BATCH_SCRIPT "Queue \n";
230  close(BATCH_SCRIPT);
231  `condor_submit $batch_script`;
232  } #HTCondor
233 
234  # slurm case
235  if($batch_system eq 'slurm') {
236  $batch_script = "$filename_template.sh";
237  open(BATCH_SCRIPT, ">$batch_script") or die("Can not create the SLURM batch script");
238  print BATCH_SCRIPT "#!/bin/bash \n";
239  print BATCH_SCRIPT "#SBATCH-p $queue \n";
240  print BATCH_SCRIPT "#SBATCH-o $filename_template.lsfout.log \n";
241  print BATCH_SCRIPT "#SBATCH-e $filename_template.lsferr.log \n";
242  print BATCH_SCRIPT "source $genie_setup $config_dir\n";
243  print BATCH_SCRIPT "cd $jobs_dir \n";
244  print BATCH_SCRIPT "$evgen_cmd \n";
245  close(BATCH_SCRIPT);
246  `sbatch --job-name=$jobname $batch_script`;
247  } #slurm
248 
249  # no batch system, run jobs interactively
250  if($batch_system eq 'none') {
251  system("source $genie_setup $config_dir; cd $jobs_dir; $evgen_cmd");
252  } # interactive mode
253 
254  }#subrunnu
255 
256  }
257 }
Generated by   doxygen 1.8.11