submit_vA_xsec_calc_jobs.pl
Go to the documentation of this file.
1 #!/usr/bin/perl
2 
3 #----------------------------------------------------------------------------------------------------------------
4 # Submit jobs for calculating GENIE x-section splines for all nuclear targets and at the energy range required
5 # for generating the GENIE release validation samples. Note that other scripts are available for generating
6 # x-section splines for the larger array of targets found in detector geometry descriptions of given expts.
7 # If needed, use the GENIE gspladd utility to merge the job outputs.
8 #
9 # Syntax:
10 # shell% perl submit_vA_xsec_calc_jobs.pl <options>
11 #
12 # Options:
13 # --version : GENIE version number
14 # --config-file : Text file which specifies the list of initial states
15 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
16 # [--production] : default: routine_validation
17 # [--cycle] : default: 01
18 # [--use-valgrind] : default: off
19 # [--batch-system] : <PBS, LSF, slurm, HTCondor, HTCondor_PBS, none>, default: PBS
20 # [--queue] : default: prod
21 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
22 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/scratch/GENIE/
23 #
24 # Author:
25 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
26 # University of Liverpool & STFC Rutherford Appleton Laboratory
27 #
28 # Copyright:
29 # Copyright (c) 2003-2020, The GENIE Collaboration
30 # For the full text of the license visit http://copyright.genie-mc.org
31 #----------------------------------------------------------------------------------------------------------------
32 
33 use File::Path;
34 use File::Basename;
35 
36 # inputs
37 #
38 $iarg=0;
39 foreach (@ARGV) {
40  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
41  if($_ eq '--config-file') { $config_file = $ARGV[$iarg+1]; }
42  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
43  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
44  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
45  if($_ eq '--use-valgrind') { $use_valgrind = $ARGV[$iarg+1]; }
46  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
47  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
48  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
49  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
50  $iarg++;
51 }
52 die("** Aborting [Undefined GENIE version. Use the --version option]")
53 unless defined $genie_version;
54 die("** Aborting [Undefined configuration file. Use the --config-file option. See examples in ?]")
55 unless defined $config_file;
56 
57 $use_valgrind = 0 unless defined $use_valgrind;
58 $arch = "SL6.x86_64" unless defined $arch;
59 $production = "routine_validation" unless defined $production;
60 $cycle = "01" unless defined $cycle;
61 $batch_system = "PBS" unless defined $batch_system;
62 $queue = "prod" unless defined $queue;
63 $softw_topdir = "/opt/ppd/t2k/softw/GENIE/" unless defined $softw_topdir;
64 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
65 $time_limit = "60:00:00";
66 
67 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
68 $freenucsplines = "$softw_topdir/data/job_inputs/xspl/gxspl-vN-$genie_version.xml";
69 $config = basename($config_file,".list");
70 $jobs_dir = "$jobs_topdir/$genie_version-$production\_$cycle-xsec\_vA\_$config";
71 
72 $nknots = 0;
73 $emax = 0;
74 @neutrinos = ();
75 @targets = ();
76 
77 open (CONFIG, $config_file);
78 $i=0;
79 while(<CONFIG>)
80 {
81  chomp;
82  # skip comment lines starting with # and empty lines that don't contain any number
83  if( substr($_,0,1) ne '#' && m/[0-9]/ )
84  {
85  s/ //g; # rm empty spaces from $_
86  #print "line = $_\n" ;
87  $value = "";
88  $idx = index($_,"#");
89  if($idx == -1) { $value = $_; }
90  else { $value = substr($_,0,$idx); }
91 
92  if ($i==0) { $emax = $value; }
93  elsif ($i==1) { $nknots = $value; }
94  else {
95  if( $value == 12 || $value == 14 || $value == 16 ||
96  $value == -12 || $value == -14 || $value == -16 )
97  {
98  push(@neutrinos, $value);
99  }
100  else
101  {
102  push(@targets, $value);
103  }
104  }
105  $i++;
106  }
107 }
108 close(CONFIG);
109 
110 # make the jobs directory
111 #
112 mkpath ($jobs_dir, {verbose => 1, mode=>0777});
113 
114 $nu_codes = "";
115 $i=0;
116 foreach (@neutrinos)
117 {
118  s/ //g; # rm empty spaces from $_
119  if($i == 0) { $nu_codes = $_; }
120  else { $nu_codes = $nu_codes . ",$_"; }
121  $i++;
122 }
123 
124 #
125 # loop over nuclear targets & submit jobs
126 #
127 foreach(@targets)
128 {
129  s/ //g; # rm empty spaces from $_
130  $tgt_code = $_;
131  $jobname = "vAxscalc-$tgt_code";
132  $filename_template = "$jobs_dir/$jobname";
133  $grep_pipe = "grep -B 100 -A 30 -i \"warn\\|error\\|fatal\"";
134  $gmkspl_opt = "-p $nu_codes -t $_ -n $nknots -e $emax --input-cross-sections $freenucsplines --output-cross-sections gxspl_$tgt_code.xml";
135  $gmkspl_cmd = "gmkspl $gmkspl_opt | $grep_pipe &> $filename_template.mkspl.log";
136  print "@@ exec: $gmkspl_cmd \n";
137 
138  #
139  # submit
140  #
141 
142  # PBS case
143  if($batch_system eq 'PBS' || $batch_system eq 'HTCondor_PBS') {
144  $batch_script = "$filename_template.pbs";
145  open(PBS, ">$batch_script") or die("Can not create the PBS batch script");
146  print PBS "#!/bin/bash \n";
147  print PBS "#PBS -N $jobname \n";
148  print PBS "#PBS -o $filename_template.pbsout.log \n";
149  print PBS "#PBS -e $filename_template.pbserr.log \n";
150  print PBS "source $genie_setup \n";
151  print PBS "cd $jobs_dir \n";
152  print PBS "$gmkspl_cmd \n";
153  close(PBS);
154  $job_submission_command = "qsub";
155  if($batch_system eq 'HTCondor_PBS') {
156  $job_submission_command = "condor_qsub";
157  }
158  `$job_submission_command -q $queue $batch_script`;
159  } #PBS
160 
161  # LSF case
162  if($batch_system eq 'LSF') {
163  $batch_script = "$filename_template.sh";
164  open(LSF, ">$batch_script") or die("Can not create the LSF batch script");
165  print LSF "#!/bin/bash \n";
166  print LSF "#BSUB-j $jobname \n";
167  print LSF "#BSUB-o $filename_template.lsfout.log \n";
168  print LSF "#BSUB-e $filename_template.lsferr.log \n";
169  print LSF "source $genie_setup \n";
170  print LSF "cd $jobs_dir \n";
171  print LSF "$gmkspl_cmd \n";
172  close(LSF);
173  `bsub < $batch_script`;
174  } #LSF
175 
176  # HTCondor
177  if($batch_system eq 'HTCondor') {
178  $batch_script = "$filename_template.htc";
179  open(HTC, ">$batch_script") or die("Can not create the Condor submit description file: $batch_script");
180  print HTC "Universe = vanilla \n";
181  print HTC "Executable = $softw_topdir/generator/builds/$arch/$genie_version/src/scripts/production/batch/htcondor_exec.sh \n";
182  print HTC "Arguments = $genie_setup $jobs_dir $gmkspl_cmd \n";
183  print HTC "Log = $filename_template.log \n";
184  print HTC "Output = $filename_template.out \n";
185  print HTC "Error = $filename_template.err \n";
186  print HTC "Request_memory = 2 GB \n";
187  print HTC "Queue \n";
188  close(HTC);
189  `condor_submit $batch_script`;
190  } #HTCondor
191 
192  # slurm case
193  if($batch_system eq 'slurm') {
194  my $time_lim = `sinfo -h -p batch -o %l`;
195  my ($days, $hours, $remainder) = $time_lim =~ /([0]+)-([0-9]+):(.*)/;
196  my $newhours = $days * 24 + $hours;
197  my $new_time_lim = "$newhours:$remainder";
198  $time_limit = $new_time_lim lt $time_limit ? $new_time_lim : $time_limit;
199  $batch_script = "$filename_template.sh";
200  open(SLURM, ">$batch_script") or die("Can not create the SLURM batch script");
201  print SLURM "#!/bin/bash \n";
202  print SLURM "#SBATCH-p $queue \n";
203  print SLURM "#SBATCH-o $filename_template.lsfout.log \n";
204  print SLURM "#SBATCH-e $filename_template.lsferr.log \n";
205  print SLURM "#SBATCH-t $time_limit \n";
206  print SLURM "source $genie_setup \n";
207  print SLURM "cd $jobs_dir \n";
208  print SLURM "$gmkspl_cmd \n";
209  close(SLURM);
210  `sbatch --job-name=$jobname $batch_script`;
211  } #slurm
212 
213  # run interactively
214  if($batch_system eq 'none') {
215  system("source $genie_setup; cd $jobs_dir; $gmkspl_cmd");
216  }
217 }
218 
Generated by   doxygen 1.8.11