submit_vN_xsec_calc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #----------------------------------------------------------------------------------------------------------------
4 # Submit jobs for calculating GENIE free-nucleon cross-section splines which can then be re-used for calculating
5 # nuclear cross-sections. If needed, use the GENIE gspladd utility to merge the job outputs.
6 #
7 # Syntax:
8 # shell% perl submit_vN_xsec_calc_jobs.pl <options>
9 #
10 # Options:
11 # --version : genie version number
12 # --xsplset : set of splines to generate
13 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
14 # [--production] : default: routine_validation
15 # [--cycle] : default: 01
16 # [--use-valgrind] : default: off
17 # [--batch-system] : <PBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: PBS
18 # [--queue] : default: prod
19 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
20 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/softw/scratch/GENIE/
21 #
22 # Examples:
23 # shell% perl submit_vN_xsec_calc_jobs.pl --version v2.6.0 --xsplset chm
24 # shell% perl submit_vN_xsec_calc_jobs.pl --version v2.6.0 --xsplset chm,nue,qel
25 # shell% perl submit_vN_xsec_calc_jobs.pl --version v2.6.0 --xsplset all
26 #
27 # Author:
28 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
29 # University of Liverpool & STFC Rutherford Appleton Laboratory
30 #
31 # Copyright:
32 # Copyright (c) 2003-2020, The GENIE Collaboration
33 # For the full text of the license visit http://copyright.genie-mc.org
34 #-------------------------------------------------------------------------------------------------------------
35 
36 use File::Path;
37 
38 # inputs
39 #
40 $iarg=0;
41 foreach (@ARGV) {
42  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
43  if($_ eq '--xsplset') { $xsplset = $ARGV[$iarg+1]; }
44  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
45  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
46  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
47  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
48  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
49  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
50  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
51  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
52  $iarg++;
53 }
54 die("** Aborting [Undefined set of cross section splines #. Use the --xsplset option]")
55 unless defined $xsplset;
56 die("** Aborting [Undefined GENIE version. Use the --version option]")
57 unless defined $genie_version;
58 
59 $use_valgrind = 0 unless defined $use_valgrind;
60 $arch = "SL6.x86_64" unless defined $arch;
61 $production = "routine_validation" unless defined $production;
62 $cycle = "01" unless defined $cycle;
63 $batch_system = "PBS" unless defined $batch_system;
64 $queue = "prod" unless defined $queue;
65 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
66 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
67 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
68 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-xsec\_vN/";
69 $time_limit = "60:00:00";
70 
71 $nkots = 500;
72 $emax = 500;
73 
74 %NUPDG = ( 'chm' => '12,-12,14,-14,16,-16',
75  'nue' => '12,-12,14,-14,16,-16',
76  'qel' => '12,-12,14,-14,16,-16',
77  'dis_ebar_cc' => '-12',
78  'dis_ebar_nc' => '-12',
79  'dis_e_cc' => '12',
80  'dis_e_nc' => '12',
81  'dis_mubar_cc' => '-14',
82  'dis_mubar_nc' => '-14',
83  'dis_mu_cc' => '14',
84  'dis_mu_nc' => '14',
85  'dis_taubar_cc' => '-16',
86  'dis_taubar_nc' => '-16',
87  'dis_tau_cc' => '16',
88  'dis_tau_nc' => '16',
89  'res_ebar_cc' => '-12',
90  'res_ebar_nc' => '-12',
91  'res_e_cc' => '12',
92  'res_e_nc' => '12',
93  'res_mubar_cc' => '-14',
94  'res_mubar_nc' => '-14',
95  'res_mu_cc' => '14',
96  'res_mu_nc' => '14',
97  'res_taubar_cc' => '-16',
98  'res_taubar_nc' => '-16',
99  'res_tau_cc' => '16',
100  'res_tau_nc' => '16' );
101 
102 %TGTPDG = ( 'chm' => '1000010010,1000000010',
103  'nue' => '1000010010,1000000010',
104  'qel' => '1000010010,1000000010',
105  'dis_ebar_cc' => '1000010010,1000000010',
106  'dis_ebar_nc' => '1000010010,1000000010',
107  'dis_e_cc' => '1000010010,1000000010',
108  'dis_e_nc' => '1000010010,1000000010',
109  'dis_mubar_cc' => '1000010010,1000000010',
110  'dis_mubar_nc' => '1000010010,1000000010',
111  'dis_mu_cc' => '1000010010,1000000010',
112  'dis_mu_nc' => '1000010010,1000000010',
113  'dis_taubar_cc' => '1000010010,1000000010',
114  'dis_taubar_nc' => '1000010010,1000000010',
115  'dis_tau_cc' => '1000010010,1000000010',
116  'dis_tau_nc' => '1000010010,1000000010',
117  'res_ebar_cc' => '1000010010,1000000010',
118  'res_ebar_nc' => '1000010010,1000000010',
119  'res_e_cc' => '1000010010,1000000010',
120  'res_e_nc' => '1000010010,1000000010',
121  'res_mubar_cc' => '1000010010,1000000010',
122  'res_mubar_nc' => '1000010010,1000000010',
123  'res_mu_cc' => '1000010010,1000000010',
124  'res_mu_nc' => '1000010010,1000000010',
125  'res_taubar_cc' => '1000010010,1000000010',
126  'res_taubar_nc' => '1000010010,1000000010',
127  'res_tau_cc' => '1000010010,1000000010',
128  'res_tau_nc' => '1000010010,1000000010' );
129 
130 %GEVGL = ( 'chm' => 'Charm',
131  'nue' => 'NuE',
132  'qel' => 'QE',
133  'dis_ebar_cc' => 'CCDIS',
134  'dis_ebar_nc' => 'NCDIS',
135  'dis_e_cc' => 'CCDIS',
136  'dis_e_nc' => 'NCDIS',
137  'dis_mubar_cc' => 'CCDIS',
138  'dis_mubar_nc' => 'NCDIS',
139  'dis_mu_cc' => 'CCDIS',
140  'dis_mu_nc' => 'NCDIS',
141  'dis_taubar_cc' => 'CCDIS',
142  'dis_taubar_nc' => 'NCDIS',
143  'dis_tau_cc' => 'CCDIS',
144  'dis_tau_nc' => 'NCDIS',
145  'res_ebar_cc' => 'CCRES',
146  'res_ebar_nc' => 'NCRES',
147  'res_e_cc' => 'CCRES',
148  'res_e_nc' => 'NCRES',
149  'res_mubar_cc' => 'CCRES',
150  'res_mubar_nc' => 'NCRES',
151  'res_mu_cc' => 'CCRES',
152  'res_mu_nc' => 'NCRES',
153  'res_taubar_cc' => 'CCRES',
154  'res_taubar_nc' => 'NCRES',
155  'res_tau_cc' => 'CCRES',
156  'res_tau_nc' => 'NCRES' );
157 
158 %OUTXML = ( 'chm' => 'pgxspl-chm.xml',
159  'nue' => 'pgxspl-nue.xml',
160  'qel' => 'pgxspl-qel.xml',
161  'dis_ebar_cc' => 'pgxspl-dis_ebar_cc.xml',
162  'dis_ebar_nc' => 'pgxspl-dis_ebar_nc.xml',
163  'dis_e_cc' => 'pgxspl-dis_e_cc.xml',
164  'dis_e_nc' => 'pgxspl-dis_e_nc.xml',
165  'dis_mubar_cc' => 'pgxspl-dis_mubar_cc.xml',
166  'dis_mubar_nc' => 'pgxspl-dis_mubar_nc.xml',
167  'dis_mu_cc' => 'pgxspl-dis_mu_cc.xml',
168  'dis_mu_nc' => 'pgxspl-dis_mu_nc.xml',
169  'dis_taubar_cc' => 'pgxspl-dis_taubar_cc.xml',
170  'dis_taubar_nc' => 'pgxspl-dis_taubar_nc.xml',
171  'dis_tau_cc' => 'pgxspl-dis_tau_cc.xml',
172  'dis_tau_nc' => 'pgxspl-dis_tau_nc.xml',
173  'res_ebar_cc' => 'pgxspl-res_ebar_cc.xml',
174  'res_ebar_nc' => 'pgxspl-res_ebar_nc.xml',
175  'res_e_cc' => 'pgxspl-res_e_cc.xml',
176  'res_e_nc' => 'pgxspl-res_e_nc.xml',
177  'res_mubar_cc' => 'pgxspl-res_mubar_cc.xml',
178  'res_mubar_nc' => 'pgxspl-res_mubar_nc.xml',
179  'res_mu_cc' => 'pgxspl-res_mu_cc.xml',
180  'res_mu_nc' => 'pgxspl-res_mu_nc.xml',
181  'res_taubar_cc' => 'pgxspl-res_taubar_cc.xml',
182  'res_taubar_nc' => 'pgxspl-res_taubar_nc.xml',
183  'res_tau_cc' => 'pgxspl-res_tau_cc.xml',
184  'res_tau_nc' => 'pgxspl-res_tau_nc.xml' );
185 
186 # make the jobs directory
187 #
188 print "@@ Creating job directory: $jobs_dir \n";
189 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
190 
191 for my $curr_xsplset (keys %OUTXML) {
192  if($xsplset=~m/$curr_xsplset/ || $xsplset eq "all") {
193 
194  #
195  # get runnu-dependent info
196  #
197  $nu = $NUPDG {$curr_xsplset};
198  $tgt = $TGTPDG {$curr_xsplset};
199  $gevgl = $GEVGL {$curr_xsplset};
200  $outxml = $OUTXML {$curr_xsplset};
201 
202  $jobname = "vNxscalc-$curr_xsplset";
203  $filename_template = "$jobs_dir/$jobname";
204 
205  $grep_pipe = "grep -B 100 -A 30 -i \"warn\\|error\\|fatal\"";
206  $valgrind_cmd = "valgrind --tool=memcheck --error-limit=no --leak-check=yes --show-reachable=yes";
207  $gmkspl_opt = "-p $nu -t $tgt -n $nkots -e $emax -o $outxml --event-generator-list $gevgl";
208  $gmkspl_cmd = "gmkspl $gmkspl_opt";
209 
210  print "@@ exec: $gmkspl_cmd \n";
211 
212  #
213  # submit
214  #
215 
216  # PBS case
217  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
218  $batch_script = "$filename_template.pbs";
219  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
220  print PBS "#!/bin/bash \n";
221  print PBS "#PBS -N $jobname \n";
222  print PBS "#PBS -o $filename_template.pbsout.log \n";
223  print PBS "#PBS -e $filename_template.pbserr.log \n";
224  print PBS "source $genie_setup \n";
225  print PBS "cd $jobs_dir \n";
226  print PBS "$gmkspl_cmd | $grep_pipe &> $filename_template.mkspl.log \n";
227  close(PBS);
228  $job_submission_command = "qsub";
229  if($batch_system eq 'HTCondor_PBS') {
230  $job_submission_command = "condor_qsub";
231  }
232  `$job_submission_command -q $queue $batch_script`;
233  } #PBS / #HTCondor_PBS
234 
235  # LSF case
236  if($batch_system eq 'LSF') {
237  $batch_script = "$filename_template.sh";
238  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
239  print LSF "#!/bin/bash \n";
240  print PBS "#BSUB-j $jobname \n";
241  print LSF "#BSUB-q $queue \n";
242  print LSF "#BSUB-o $filename_template.lsfout.log \n";
243  print LSF "#BSUB-e $filename_template.lsferr.log \n";
244  print LSF "source $genie_setup \n";
245  print LSF "cd $jobs_dir \n";
246  print LSF "$gmkspl_cmd | $grep_pipe &> $filename_template.mkspl.log \n";
247  close(LSF);
248  `bsub < $batch_script`;
249  } #LSF
250 
251  # HTCondor
252  if($batch_system eq 'HTCondor') {
253  $batch_script = "$filename_template.htc";
254  open(HTC, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
255  print HTC "Universe = vanilla \n";
256  print HTC "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
257  print HTC "Arguments = $genie_setup $jobs_dir $gmkspl_cmd \n";
258  print HTC "Log = $filename_template.log \n";
259  print HTC "Output = $filename_template.out \n";
260  print HTC "Error = $filename_template.err \n";
261  print HTC "Request_memory = 2 GB \n";
262  print HTC "Queue \n";
263  close(HTC);
264  `condor_submit $batch_script`;
265  } #HTCondor
266 
267  # slurm case
268  if($batch_system eq 'slurm') {
269  my $time_lim = `sinfo -h -p batch -o %l`;
270  my ($days, $hours, $remainder) = $time_lim =~ /([0]+)-([0-9]+):(.*)/;
271  my $newhours = $days * 24 + $hours;
272  my $new_time_lim = "$newhours:$remainder";
273  $time_limit = $new_time_lim lt $time_limit ? $new_time_lim : $time_limit;
274  $batch_script = "$filename_template.sh";
275  open(SLURM, ">$batch_script") or die("Can not create the slurm batch script");
276  print SLURM "#!/bin/bash \n";
277  print SLURM "#SBATCH-p $queue \n";
278  print SLURM "#SBATCH-o $filename_template.slurmout.log \n";
279  print SLURM "#SBATCH-e $filename_template.slurmerr.log \n";
280  print SLURM "#SBATCH-t $time_limit \n";
281  print SLURM "source $genie_setup \n";
282  print SLURM "cd $jobs_dir \n";
283  print SLURM "$gmkspl_cmd | $grep_pipe &> $filename_template.mkspl.log \n";
284  close(SLURM);
285  `sbatch --job-name=$jobname $batch_script`;
286  } #slurm
287 
288  # no batch system, run jobs interactively
289  if($batch_system eq 'none') {
290  system("source $genie_setup; cd $jobs_dir; $gmkspl_cmd");
291  } # interactive mode
292 
293 
294  }
295 }
Generated by   doxygen 1.8.11