run_genie_validation.pl
Go to the documentation of this file.
1 #---------------------------------------------------------------------------------------
2 # Automate running of all routine GENIE MC validation tasks.
3 #
4 # Use the cron daemon to run this script every few minutes or so.
5 # It will monitor the progress, submit jobs as necessary, assemble the data outputs,
6 # run the validation checks and summarize the generator status.
7 #
8 # Options:
9 # --version : genie version number
10 # [--user] : username, used for monitoring job progress, default: candreop
11 # [--ref-data-topdir] : path to output files from a previous production used for reference
12 # [--ref-data-label] :
13 # [--arch] : <SL4.x86_32, SL5.x86_64, SL6.x86_64, ...>, default: SL6.x86_64
14 # [--production] : default: routine_validation
15 # [--cycle] : default: 01
16 # [--batch-system] : <PBS, LSF>, default: PBS (currently works only for PBS)
17 # [--queue] : default: prod
18 # [--softw-topdir] : top level dir for softw installations, default: /opt/ppd/t2k/softw/GENIE/
19 # [--jobs-topdir] : top level dir for job files, default: /opt/ppd/t2k/softw/scratch/GENIE/
20 #
21 # Author:
22 # Costas Andreopoulos <costas.andreopoulos \st stfc.ac.uk>
23 # University of Liverpool & STFC Rutherford Appleton Laboratory
24 #
25 # Copyright:
26 # Copyright (c) 2003-2020, The GENIE Collaboration
27 # For the full text of the license visit http://copyright.genie-mc.org
28 #---------------------------------------------------------------------------------------
29 
30 #!/usr/bin/perl
31 
32 use File::Path;
33 
34 foreach (@ARGV) {
35  if($_ eq '--version') { $genie_version = $ARGV[$iarg+1]; }
36  if($_ eq '--user') { $user = $ARGV[$iarg+1]; }
37  if($_ eq '--ref-data-topdir') { $ref_topdir = $ARGV[$iarg+1]; }
38  if($_ eq '--ref-data-label') { $ref_label = $ARGV[$iarg+1]; }
39  if($_ eq '--arch') { $arch = $ARGV[$iarg+1]; }
40  if($_ eq '--production') { $production = $ARGV[$iarg+1]; }
41  if($_ eq '--cycle') { $cycle = $ARGV[$iarg+1]; }
42  if($_ eq '--batch-system') { $batch_system = $ARGV[$iarg+1]; }
43  if($_ eq '--queue') { $queue = $ARGV[$iarg+1]; }
44  if($_ eq '--softw-topdir') { $softw_topdir = $ARGV[$iarg+1]; }
45  if($_ eq '--jobs-topdir') { $jobs_topdir = $ARGV[$iarg+1]; }
46  $iarg++;
47 }
48 die("** Aborting [Undefined GENIE version. Use the --version option]")
49 unless defined $genie_version;
50 
51 $user = "candreop" unless defined $user;
52 $ref_label = "reference" unless defined $ref_label;
53 $arch = "SL6.x86_64" unless defined $arch;
54 $production = "routine_validation" unless defined $production;
55 $cycle = "01" unless defined $cycle;
56 $batch_system = "PBS" unless defined $batch_system;
57 $queue = "prod" unless defined $queue;
58 $softw_topdir = "/opt/ppd/t2k/softw/GENIE" unless defined $softw_topdir;
59 $jobs_topdir = "/opt/ppd/t2k/scratch/GENIE/" unless defined $jobs_topdir;
60 
61 $out_data_dir = "$softw_topdir/data/stage/$genie_version-$production\_$cycle";
62 $inp_data_dir = "$softw_topdir/data/job_inputs/";
63 $genie_setup = "$softw_topdir/generator/builds/$arch/$genie_version-setup";
64 $genie_topdir = "$softw_topdir/generator/builds/$arch/$genie_version/";
65 $scripts_dir = "$genie_topdir/src/scripts/production/batch/";
66 $scripts_dir2 = "$genie_topdir/src/scripts/production/misc/";
67 $status_file = "$jobs_topdir/$genie_version-$production\_$cycle.status";
68 
69 $std_args = "--version $genie_version " .
70  "--arch $arch " .
71  "--softw-topdir $softw_topdir " .
72  "--production $production " .
73  "--cycle $cycle " .
74  "--batch-system $batch_system " .
75  "--queue $queue";
76 
77 init_status_file($status_file);
78 
79 $status = read_status_file($status_file);
80 print "Current status: $status \n";
81 
82 #
83 # Create data directories
84 #
85 if($status eq "starting")
86 {
87  print "Need to create directory structure for output data \n";
88  mkpath ( $out_data_dir, {verbose => 1, mode=>0777});
89  mkpath ("$out_data_dir/xsec/", {verbose => 1, mode=>0777});
90  mkpath ("$out_data_dir/mctest/", {verbose => 1, mode=>0777});
91  mkpath ("$out_data_dir/mctest/ghep/", {verbose => 1, mode=>0777});
92  mkpath ("$out_data_dir/mctest/gst/", {verbose => 1, mode=>0777});
93  mkpath ("$out_data_dir/xsec_validation/", {verbose => 1, mode=>0777});
94  mkpath ("$out_data_dir/xsec_validation/ghep/", {verbose => 1, mode=>0777});
95  mkpath ("$out_data_dir/xsec_validation/gst/", {verbose => 1, mode=>0777});
96  mkpath ("$out_data_dir/hadronization/", {verbose => 1, mode=>0777});
97  mkpath ("$out_data_dir/hadronization/ghep/", {verbose => 1, mode=>0777});
98  mkpath ("$out_data_dir/intranuke/", {verbose => 1, mode=>0777});
99  mkpath ("$out_data_dir/intranuke/ghep/", {verbose => 1, mode=>0777});
100  mkpath ("$out_data_dir/reptest/", {verbose => 1, mode=>0777});
101  mkpath ("$out_data_dir/reptest/ghep/", {verbose => 1, mode=>0777});
102  mkpath ("$out_data_dir/flux/", {verbose => 1, mode=>0777});
103  mkpath ("$out_data_dir/geom/", {verbose => 1, mode=>0777});
104  mkpath ("$out_data_dir/reweight/", {verbose => 1, mode=>0777});
105  mkpath ("$out_data_dir/reports/", {verbose => 1, mode=>0777});
106  update_status_file($status_file,"done creating directory structure for output data");
107  exit;
108 }
109 
110 #
111 # ......................................................................................
112 # Calculate neutrino-nucleon cross-section splines for use in subsequent calculations
113 # and MC sample generation jobs. Compare the calculated neutrino-nucleon cross-sections
114 # with reference calculations and generate report.
115 # ......................................................................................
116 #
117 
118 #
119 # Run jobs needed
120 #
121 if($status eq "done creating directory structure for output data")
122 {
123  print "Need to produce free-nucleon cross-section splines \n";
124  $cmd = "perl $scripts_dir/submit_vN_xsec_calc_jobs.pl $std_args --xsplset all";
125  execute_command($cmd);
126  update_status_file($status_file,"calculating neutrino-nucleon cross-section splines");
127  exit;
128 }
129 
130 #
131 # Once all jobs are done, check for errors and merge all free-nucleon cross-section xml files
132 #
133 if($status eq "calculating neutrino-nucleon cross-section splines")
134 {
135  $njobs = num_of_jobs_running($user,"vNxscalc");
136  if($njobs > 0) {
137  exit;
138  }
139  print "No neutrino-nucleon cross-section calculation job still running... \n";
140 
141  # Make sure there is one output XML file for each input PBS script
142  print "Making sure all output files are present... \n";
143  my $ret = check_number_of_xml_and_pbs_files("$jobs_topdir/$genie_version-$production\_$cycle-xsec\_vN/");
144  if($ret != 0) {
145  print "The number of output XML files does not match the number of input PBS files. \n";
146  print "Some of the batch jobs have failed. \n";
147  print "Can not continue with validation runs unless this is sorted out.\n";
148  exit;
149  }
150 
151  # Check log files for errors
152  print "Checking for errors... \n";
153  $nerr = `more $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vN/*log | grep -i error | wc -l`
154  + `more $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vN/*log | grep -i fatal | wc -l`;
155  if($nerr > 0)
156  {
157  print "I found $nerr errors in the neutrino-nucleon cross-section calculation job log-files. \n";
158  print "Can not continue with validation runs unless this is sorted out.\n";
159  exit;
160  }
161 
162  # Merge all XML files to one and copy to standard locations
163  print "Merging all free nucleon cross-section XML files... \n";
164  $cmd = "source $genie_setup; " .
165  "gspladd -d $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vN/ -o gxspl-vN-$genie_version.xml; " .
166  "cp gxspl-vN-$genie_version.xml $inp_data_dir/xspl/; " .
167  "cp gxspl-vN-$genie_version.xml $out_data_dir/xsec/; " .
168  "rm -f gxspl-vN-$genie_version.xml";
169  execute_command($cmd);
170  update_status_file($status_file,"done calculating neutrino-nucleon cross-section splines");
171  exit;
172 }
173 
174 #
175 # Convert calculated cross-sections from XML to ROOT format
176 #
177 if($status eq "done calculating neutrino-nucleon cross-section splines")
178 {
179  print "Converting all free-nucleon cross-section files from XML to ROOT format... \n";
180  $batch_cmd = "source $genie_setup; " .
181  "gspl2root -p 12,-12,14,-14,16,-16 -t 1000010010,1000000010 -o xsec.root -f $inp_data_dir/xspl/gxspl-vN-$genie_version.xml; " .
182  "cp xsec.root $out_data_dir/xsec/";
183  $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name xsconv $std_args";
184  execute_command($cmd);
185  update_status_file($status_file,"converting neutrino-nucleon cross-section data to ROOT format");
186  exit;
187 }
188 
189 #
190 # Compare calculated cross-sections with reference cross-sections and generate report
191 #
192 if($status eq "converting neutrino-nucleon cross-section data to ROOT format")
193 {
194  $njobs = num_of_jobs_running($user,"xsconv");
195  if($njobs > 0) {
196  exit;
197  }
198  if (defined $ref_topdir)
199  {
200  print "Comparing free-nucleon cross-sections with reference calculation... \n";
201  $batch_cmd = "source $genie_setup; " .
202  "gvld_xsec_comp -f $out_data_dir/xsec/xsec.root,$genie_version -r $ref_topdir/xsec/xsec.root,$ref_label -o xsec.ps; " .
203  "ps2pdf14 xsec.ps; " .
204  "cp xsec.pdf $out_data_dir/reports/";
205  $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name xscomp $std_args";
206  execute_command($cmd);
207  }
208  update_status_file($status_file,"comparing neutrino-nucleon cross-sections with reference calculations");
209  exit;
210 }
211 
212 #
213 # Check last report and make sure it is OK to continue
214 #
215 if($status eq "comparing neutrino-nucleon cross-sections with reference calculations")
216 {
217  $njobs = num_of_jobs_running($user,"xscomp");
218  if($njobs > 0) {
219  exit;
220  }
221 
222  # ...
223  # ... check report
224  # ...
225 
226  update_status_file($status_file,"done comparing neutrino-nucleon cross-sections with reference calculations");
227  exit;
228 }
229 
230 #
231 # ......................................................................................
232 # Calculate nuclear cross-sections needed for validation MC runs
233 # ......................................................................................
234 #
235 
236 #
237 # Run jobs needed
238 #
239 if($status eq "done comparing neutrino-nucleon cross-sections with reference calculations")
240 {
241  print "Need to produce nuclear cross-section splines \n";
242  $cmd = "perl $scripts_dir/submit_vA_xsec_calc_jobs.pl $std_args --config-file $scripts_dir/xsec_splines/genie_test.list";
243  execute_command($cmd);
244  update_status_file($status_file,"calculating neutrino-nucleus cross-section splines");
245  exit;
246 }
247 
248 #
249 # Once all jobs are done, merge all nuclear cross-section xml files
250 #
251 if($status eq "calculating neutrino-nucleus cross-section splines")
252 {
253  $njobs = num_of_jobs_running($user,"vAxscalc");
254  if($njobs > 0) {
255  exit;
256  }
257  print "No neutrino-nucleus cross-section calculation job still running... \n";
258 
259  # Make sure there is one output XML file for each input PBS script
260  print "Making sure all output files are present... \n";
261  my $ret = check_number_of_xml_and_pbs_files("$jobs_topdir/$genie_version-$production\_$cycle-xsec\_vA\_genie\_test/");
262  if($ret != 0) {
263  print "The number of output XML files does not match the number of input PBS files. \n";
264  print "Some of the batch jobs have failed. \n";
265  print "Can not continue with validation runs unless this is sorted out.\n";
266  exit;
267  }
268 
269  # Check log files for errors
270  print "Checking for errors... \n";
271  $nerr = `more $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vA\_genie\_test/*log | grep -i error | wc -l`
272  + `more $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vA\_genie\_test/*log | grep -i fatal | wc -l`;
273  if($nerr >= 0)
274  {
275  print "I found $nerr errors in the neutrino-nucleus cross-section calculation job log-files. \n";
276  print "Can not continue with validation runs unless this is sorted out.\n";
277  # ------------
278  # Lots of errors are printed our by numerical integrators but things look ok
279  # Disable exit for now since we will soon be removing all the old numerical algorithms
280  # ------------
281  # exit;
282  }
283 
284  # Merge all XML files to one and copy to standard locations
285  print "Merging all neutrino-nucleus cross-section XML files... \n";
286  $cmd = "source $genie_setup; " .
287  "gspladd -d $jobs_topdir/$genie_version-$production\_$cycle-xsec\_vA\_genie\_test/ -o gxspl-vA-$genie_version.xml; " .
288  "cp gxspl-vA-$genie_version.xml $inp_data_dir/xspl/; " .
289  "cp gxspl-vA-$genie_version.xml $out_data_dir/xsec/; " .
290  "rm -f gxspl-vA-$genie_version.xml";
291  execute_command($cmd);
292  update_status_file($status_file,"done calculating neutrino-nucleus cross-section splines");
293  exit;
294 }
295 
296 #
297 # ......................................................................................
298 # Submit test MC runs, scan the log files, run sanity checks on the event samples
299 # and compare with reference samples.
300 # ......................................................................................
301 #
302 
303 #
304 # Run jobs needed
305 #
306 if($status eq "done calculating neutrino-nucleus cross-section splines")
307 {
308  print "Generating standard neutrino MC jobs \n";
309  $cmd = "perl $scripts_dir/submit_standard_neutrino_mc_test_jobs.pl $std_args --run all";
310  execute_command($cmd);
311  update_status_file($status_file,"running standard neutrino MC jobs");
312  exit;
313 }
314 
315 #
316 # Once test MC runs are completed, move all data files from the scratch area
317 #
318 if($status eq "running standard neutrino MC jobs")
319 {
320  $njobs = num_of_jobs_running($user,"mctest");
321  if($njobs > 0) {
322  exit;
323  }
324 
325  #
326  # Check log files for errors and make sure there is one output ROOT file for each input PBS script
327  #
328  print "Checking for errors... \n";
329  $nerr = `more $jobs_topdir/$genie_version-$production\_$cycle-mctest/*log | grep -i error | wc -l`
330  + `more $jobs_topdir/$genie_version-$production\_$cycle-mctest/*log | grep -i fatal | wc -l`;
331  if($nerr > 0)
332  {
333  print "I found $nerr errors in the standard neutrino MC job log-files. \n";
334  print "Can not continue with validation runs unless this is sorted out.\n";
335 # exit;
336  }
337  print "Checking the number of output ROOT/GHEP files... \n";
338  opendir my $dir, "$jobs_topdir/$genie_version-$production\_$cycle-mctest/" or die "Can not open directory: $!";
339  my @evtfiles = grep { /.ghep.root$/ } readdir $dir;
340  rewinddir $dir;
341  my @pbsfiles = grep { /.pbs$/ } readdir $dir;
342  closedir $dir;
343  $num_evtfiles = @evtfiles;
344  $num_pbsfiles = @pbsfiles;
345  print "Found $num_evtfiles ROOT/GHEP event files and $num_pbsfiles PBS files. \n";
346  if($num_evtfiles != $num_pbsfiles)
347  {
348  print "The number of output event files doesn't match the number of input PBS files. \n";
349  print "Can not continue with validation runs unless this is sorted out.\n";
350  exit;
351  }
352 
353  $cmd = "mv $jobs_topdir/$genie_version-$production\_$cycle-mctest/*ghep.root $out_data_dir/mctest/ghep/; " .
354  "mv $jobs_topdir/$genie_version-$production\_$cycle-mctest/*gst.root $out_data_dir/mctest/gst/";
355  execute_command($cmd);
356  update_status_file($status_file,"done running standard neutrino MC jobs");
357  exit;
358 }
359 
360 #
361 # Run sanity checks on the test MC runs and generate reports
362 #
363 if($status eq "done running standard neutrino MC jobs")
364 {
365  print "Running sanity checks on the outputs of the the standard neutrino MC jobs ... \n";
366  opendir my $dir, "$out_data_dir/mctest/ghep/" or die "Can not open directory: $!";
367  my @files = grep { !/^\./ } readdir $dir;
368  closedir $dir;
369  $ijob = 0;
370  foreach(@files) {
371  $batch_cmd =
372  "source $genie_setup; " .
373  "gvld_sample_scan -f $out_data_dir/mctest/ghep/$_ -o $_.log " .
374  "--add-event-printout-in-error-log --event-record-print-level 2 --max-num-of-errors-shown 10 " .
375  "--check-energy-momentum-conservation " .
376  "--check-charge-conservation " .
377  "--check-for-pseudoparticles-in-final-state " .
378  "--check-for-off-mass-shell-particles-in-final-state " .
379  "--check-for-num-of-final-state-nucleons-inconsistent-with-target " .
380  "--check-vertex-distribution " .
381  "--check-decayer-consistency; " .
382  "mv $_.log $out_data_dir/reports/";
383  $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name snchk-$ijob $std_args";
384  execute_command($cmd);
385  $ijob++;
386  }
387  update_status_file($status_file,"running sanity checks on standard neutrino MC jobs");
388  exit;
389 }
390 
391 #
392 # See the reports and make sure it is OK to continue
393 #
394 if($status eq "running sanity checks on standard neutrino MC jobs")
395 {
396  $njobs = num_of_jobs_running($user,"snchk");
397  if($njobs > 0) {
398  exit;
399  }
400 
401  print "Checking results sanity checks on the outputs of the the standard neutrino MC jobs ... \n";
402 
403  # ...
404  # ...
405  # ...
406 
407  update_status_file($status_file,"done running sanity checks on standard neutrino MC jobs");
408  exit;
409 }
410 
411 #
412 # Compare the test MC samples with samples generated with a reference version of GENIE
413 #
414 if($status eq "done running sanity checks on standard neutrino MC jobs")
415 {
416  if (defined $ref_topdir)
417  {
418  opendir my $dir, "$out_data_dir/mctest/ghep/" or die "Can not open directory: $!";
419  my @files = grep { !/^\./ } readdir $dir;
420  closedir $dir;
421 
422  $ijob=0;
423  foreach(@files) {
424  $batch_cmd =
425  "source $genie_setup; " .
426  "gvld_sample_comp -f $out_data_dir/mctest/ghep/$_ -r $ref_topdir/mctest/ghep/$_ -o x; " .
427  "mv x $out_data_dir/reports/";
428  $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name compmc-$ijob $std_args";
429  system("$cmd");
430  $ijob++;
431  }
432  }
433  update_status_file($status_file,"comparing standard neutrino MC jobs with reference samples");
434  exit;
435 }
436 
437 #
438 # ......................................................................................
439 # Repeatability test: Submit different jobs with same inputs and same seed and make sure
440 # that the generated samples are identical.
441 # ......................................................................................
442 #
443 
444 if($status eq "comparing standard neutrino MC jobs with reference samples")
445 {
446  # sample 1
447  {
448  my $batch_cmd =
449  "source $genie_setup; " .
450  "gevgen -p 14 -t 1000260560 -e 0.1,50 -f '1/x' --seed 123456 --run 100 --cross-sections $inp_data_dir/xspl/gxspl-vA-$genie_version.xml; " .
451  "mv *ghep.root $out_data_dir/reptest/ghep/";
452  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name repmc1 $std_args";
453  execute_command($cmd);
454  }
455 
456  # sample 2
457  {
458  my $batch_cmd =
459  "source $genie_setup; " .
460  "gevgen -p 14 -t 1000260560 -e 0.1,50 -f '1/x' --seed 123456 --run 101 --cross-sections $inp_data_dir/xspl/gxspl-vA-$genie_version.xml; " .
461  "mv *ghep.root $out_data_dir/reptest/ghep/";
462  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name repmc2 $std_args";
463  execute_command($cmd);
464  }
465 
466  # sample 3
467  {
468  my $batch_cmd =
469  "source $genie_setup; " .
470  "gevgen -p 14 -t 1000260560 -e 0.1,50 -f '1/x' --seed 123456 --run 102 --cross-sections $inp_data_dir/xspl/gxspl-vA-$genie_version.xml; " .
471  "mv *ghep.root $out_data_dir/reptest/ghep/";
472  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name repmc2 $std_args";
473  execute_command($cmd);
474  }
475 
476  update_status_file($status_file,"running jobs for MC repeatability test");
477  exit;
478 }
479 
480 if($status eq "running jobs for MC repeatability test")
481 {
482  $njobs = num_of_jobs_running($user,"repmc");
483  if($njobs > 0) {
484  exit;
485  }
486 
487  {
488  my $batch_cmd =
489  "source $genie_setup; " .
490  "gvld_repeatability_test --first-sample $out_data_dir/reptest/ghep/gntp.100.ghep.root " .
491  " --second-sample $out_data_dir/reptest/ghep/gntp.101.ghep.root " .
492  " --add-event-printout-in-error-log --max-num-of-errors-shown 10 -o reptest_runs100vs101.log; " .
493  "mv reptest_runs100vs101.log $out_data_dir/reports/";
494  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name reptest1 $std_args";
495  execute_command($cmd);
496  }
497 
498  {
499  my $batch_cmd =
500  "source $genie_setup; " .
501  "gvld_repeatability_test --first-sample $out_data_dir/reptest/ghep/gntp.100.ghep.root " .
502  " --second-sample $out_data_dir/reptest/ghep/gntp.102.ghep.root " .
503  " --add-event-printout-in-error-log --max-num-of-errors-shown 10 -o reptest_runs100vs102.log; " .
504  "mv reptest_runs100vs102.log $out_data_dir/reports/";
505  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name reptest2 $std_args";
506  execute_command($cmd);
507  }
508 
509  update_status_file($status_file,"running MC repeatability test");
510  exit;
511 }
512 
513 if($status eq "running MC repeatability test")
514 {
515  $njobs = num_of_jobs_running($user,"reptest");
516  if($njobs > 0) {
517  exit;
518  }
519 
520  # ...
521  # ... check log files and make sure it is OK to continue
522  # ...
523 
524  update_status_file($status_file,"done running MC repeatability test");
525  exit;
526 }
527 
528 #
529 # ......................................................................................
530 # Compare the neutrino cross-section model with world data
531 # ......................................................................................
532 #
533 
534 #
535 # Run jobs needed
536 #
537 if($status eq "done running MC repeatability test")
538 {
539  $njobs = num_of_jobs_running($user,"compmc");
540  if($njobs > 0) {
541  exit;
542  }
543  print "Generating MC samples needed for comparing the cross-section model with data \n";
544  $cmd = "perl $scripts_dir/submit_neutrino_xsec_validation_mc_jobs.pl $std_args --nsubruns 1";
545  execute_command($cmd);
546  update_status_file($status_file,"running MC jobs for cross-section model validation");
547  exit;
548 }
549 
550 #
551 # Once cross-section MC jobs are done, move all data files from the scratch area
552 #
553 if($status eq "running MC jobs for cross-section model validation")
554 {
555  $njobs = num_of_jobs_running($user,"xsecvld");
556  if($njobs > 0) {
557  exit;
558  }
559  $cmd = "mv $jobs_topdir/$genie_version-$production\_$cycle-xsec_validation/*ghep.root $out_data_dir/xsec_validation/ghep/; " .
560  "mv $jobs_topdir/$genie_version-$production\_$cycle-xsec_validation/*gst.root $out_data_dir/xsec_validation/gst/";
561  execute_command($cmd);
562  update_status_file($status_file,"done running MC jobs for cross-section model validation");
563  exit;
564 }
565 
566 #
567 # Run actual comparisons with cross-section data and compute error envelopes
568 #
569 if($status eq "done running MC jobs for cross-section model validation")
570 {
571  # Create an XML file list containing the generated MC files and cross-section files needed for data/MC comparisons
572  {
573  my $cmd = "perl $scripts_dir2/make_genie_sim_file_list.pl " .
574  "$out_data_dir/xsec_validation/ghep/:$out_data_dir/xsec/,$genie_version " .
575  "> $out_data_dir/xsec_validation/file_list.xml";
576  execute_command($cmd);
577  }
578 
579  # Create an XML file list, as above, but include the reference samples too
580  if (defined $ref_topdir)
581  {
582  my $cmd = "perl $scripts_dir2/make_genie_sim_file_list.pl " .
583  "$out_data_dir/xsec_validation/ghep/:$out_data_dir/xsec/,$genie_version " .
584  "$ref_topdir/xsec_validation/ghep/:$ref_topdir/xsec/,$ref_label " .
585  "> $out_data_dir/xsec_validation/file_list_ref.xml";
586  execute_command($cmd);
587  }
588 
589  # Submit a single job to generate all GENIE/data comparisons for the current version
590  {
591  my $batch_cmd =
592  "source $genie_setup; " .
593  "gvld_nu_xsec -g $out_data_dir/xsec_validation/file_list.xml -o genie_$genie_version-world_nu_xsec_data_comp-all; " .
594  "ps2pdf14 genie_$genie_version-world_nu_xsec_data_comp-all.ps; " .
595  "mv genie_$genie_version-world_nu_xsec_data_comp-all.pdf $out_data_dir/reports/; " .
596  "mv genie_$genie_version-world_nu_xsec_data_comp-all.root $out_data_dir/reports/; ";
597  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name xseccmp1 $std_args";
598  execute_command($cmd);
599  }
600 
601  # Submit a single job to generate all GENIE/data comparisons for the current and reference versions
602  if (defined $ref_topdir)
603  {
604  my $batch_cmd =
605  "source $genie_setup; " .
606  "gvld_nu_xsec -g $out_data_dir/xsec_validation/file_list_ref.xml -o genie_$genie_version-world_nu_xsec_data_comp-all-withref; " .
607  "ps2pdf14 genie_$genie_version-world_nu_xsec_data_comp-all-withref.ps; " .
608  "mv genie_$genie_version-world_nu_xsec_data_comp-all-withref.pdf $out_data_dir/reports/";
609  "mv genie_$genie_version-world_nu_xsec_data_comp-all-withref.root $out_data_dir/reports/";
610  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name xseccmp2 $std_args";
611  execute_command($cmd);
612  }
613 
614  #
615  # Submit jobs to generate GENIE/data comparisons for the current version
616  # and calculate error envelopes for the GENIE prediction. Because calculating error
617  # envelopes is CPU-intensive submit one job per each GENIE/data comparison.
618  #
619  my @comparisons = (
620  "numuCC_all", "numubarCC_all",
621  "numuCC_lowE", "numubarCC_lowE", "numuCC_highE", "numubarCC_highE",
622  "numuCC_minos", "numubarCC_minos", "numuCC_sciboone", "r_minos",
623  "numuCCQE_all", "numuCCQE_deuterium", "numuCCQE_heavy_target",
624  "numuCCQE_nomad_nucleon", "numuCCQE_nomad_nuclear", "numuCCQE_miniboone_nuclear", "numuCCQE_all_12C_nuclear",
625  "numubarCCQE_all", "numubarCCQE_deuterium", "numubarCCQE_heavy_target",
626  "numubarCCQE_nomad_nucleon", "numubarCCQE_nomad_nuclear",
627  "numuCCppip", "numuCCnpip", "numuCCppi0", "numuCCn2pip", "numuCCppippi0", "numuCCppippim",
628  "numuCCpi0_numuCCQE_k2k",
629  "numuNCcohpi0_Ne20", "numuCCcohpip_Ne20", "numubarCCcohpim_Ne20", "numuNCcohpi0_Al27", "numuNCcohpi0_Si30", "numuCCcohpip_Si30", "numubarCCcohpim_Si30",
630  "numuCC_dilepton_ratio_worldavg", "numubarCC_dilepton_ratio_worldavg", "numuCC_charm_ratio_worldavg",
631  "numuCC_dilepton_cdhs", "numuCC_dilepton_nomad", "numuCC_dilepton_e744_e770", "numuCC_dilepton_e744", "numuCC_dilepton_fnal15ft", "numuCC_dilepton_gargamelle"
632  );
633 
634  foreach (@comparisons) {
635  my $batch_cmd =
636  "source $genie_setup; " .
637  "gvld_nu_xsec -g $out_data_dir/xsec_validation/file_list.xml -o genie_$genie_version-world_nu_xsec_data_comp-$_; " .
638  "ps2pdf14 genie_$genie_version-world_nu_xsec_data_comp-$_.ps; " .
639  "mv genie_$genie_version-world_nu_xsec_data_comp-$_.pdf $out_data_dir/reports/";
640  "mv genie_$genie_version-world_nu_xsec_data_comp-$_.root $out_data_dir/reports/";
641  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name xseccmp-$_ $std_args";
642  execute_command($cmd);
643  }
644 
645  update_status_file($status_file,"running cross-section model comparisons with data");
646  exit;
647 }
648 
649 #
650 # ......................................................................................
651 # Compare the hadronization model with data
652 # ......................................................................................
653 #
654 
655 #
656 # Run jobs needed for comparing the hadronization model with data
657 #
658 if($status eq "running cross-section model comparisons with data")
659 {
660  $njobs = num_of_jobs_running($user,"xseccmp");
661  if($njobs > 0) {
662  exit;
663  }
664 
665  print "Generating MC samples needed for comparing the hadronization model with data \n";
666  $cmd = "perl $scripts_dir/submit_hadronization_validation_mc_jobs.pl $std_args --nsubruns 1";
667  execute_command($cmd);
668  update_status_file($status_file,"running MC jobs for hadronization model validation");
669  exit;
670 }
671 
672 #
673 # Once hadronization MC jobs are done, move all data files from the scratch area
674 #
675 if($status eq "running MC jobs for hadronization model validation")
676 {
677  $njobs = num_of_jobs_running($user,"hadro");
678  if($njobs > 0) {
679  exit;
680  }
681  $cmd = "mv $jobs_topdir/$genie_version-$production\_$cycle-hadronization/*ghep.root $out_data_dir/hadronization/ghep/";
682  execute_command($cmd);
683  update_status_file($status_file,"done running MC jobs for hadronization model validation");
684  exit;
685 }
686 
687 #
688 # Run actual comparisons with hadronization data
689 #
690 if($status eq "done running MC jobs for hadronization model validation")
691 {
692  # Create an XML file list containing the generated MC files
693  {
694  my $cmd = "perl $scripts_dir2/make_genie_sim_file_list.pl " .
695  "$out_data_dir/hadronization/ghep/,$genie_version " .
696  "> $out_data_dir/hadronization/file_list.xml";
697  execute_command($cmd);
698  }
699 
700  # Create an XML file list as above but include the reference MC files, if any.
701  if (defined $ref_topdir)
702  {
703  my $cmd = "perl $scripts_dir2/make_genie_sim_file_list.pl " .
704  "$out_data_dir/hadronization/ghep/,$genie_version " .
705  "$ref_topdir/hadronization/ghep/,$ref_label " .
706  "> $out_data_dir/hadronization/file_list_ref.xml";
707  execute_command($cmd);
708  }
709 
710  # Submit a single job to generate all GENIE/data comparisons for the current version
711  {
712  my $batch_cmd =
713  "source $genie_setup; " .
714  "gvld_hadronz_test -g $out_data_dir/hadronization/file_list.xml -o genie_$genie_version-hadronization_test.ps; " .
715  "ps2pdf14 genie_$genie_version-hadronization_test.ps; " .
716  "mv genie_$genie_version-hadronization_test.pdf $out_data_dir/reports/";
717  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name hadcmp1 $std_args";
718  execute_command($cmd);
719  }
720 
721  # Submit a single job to generate all GENIE/data comparisons for the current and reference versions
722  if (defined $ref_topdir)
723  {
724  my $batch_cmd =
725  "source $genie_setup; " .
726  "gvld_hadronz_test -g $out_data_dir/hadronization/file_list_ref.xml -o genie_$genie_version-hadronization_test-withref.ps; " .
727  "ps2pdf14 genie_$genie_version-hadronization_test-withref.ps; " .
728  "mv genie_$genie_version-hadronization_test-withref.pdf $out_data_dir/reports/";
729  my $cmd = "perl $scripts_dir/submit.pl --cmd \'$batch_cmd\' --job-name hadcmp2 $std_args";
730  execute_command($cmd);
731  }
732 
733  update_status_file($status_file,"running hadronization model comparisons with data");
734  exit;
735 }
736 
737 #
738 # ......................................................................................
739 # Compare the intranuclear rescattering model with data
740 # ......................................................................................
741 #
742 
743 
744 #...
745 #...
746 #...
747 
748 
749 #
750 # ......................................................................................
751 # Compare the structure function model with data
752 # ......................................................................................
753 #
754 
755 #...
756 #...
757 #...
758 
759 #
760 # ......................................................................................
761 # Run event reweighting tests
762 # ......................................................................................
763 #
764 
765 #...
766 #...
767 #...
768 
769 #
770 # ......................................................................................
771 # Run flux driver tests
772 # ......................................................................................
773 #
774 
775 #...
776 #...
777 #...
778 
779 #
780 # ......................................................................................
781 # Run geometry navigation tests
782 # ......................................................................................
783 #
784 
785 #...
786 #...
787 #...
788 
789 
790 
791 
792 #
793 ########################################################################################
794 ########################################################################################
795 #
796 
797 sub num_of_jobs_running
798 {
799  $user = shift;
800  $jobtag = shift;
801  @myjobs = `qstat -u $user`;
802  $njobs = 0;
803  foreach(@myjobs) {
804  chomp($_);
805  if( /$user/ && /$jobtag/ ) {
806  $njobs++;
807  print "Job still running: $_ \n";
808  }
809  }
810  print "Number of jobs still running or on the queue: $njobs \n";
811  return $njobs;
812 }
813 
814 sub init_status_file
815 {
816  $file = shift;
817  if(! -e $file)
818  {
819  open(STF,">$file");
820  print STF "starting";
821  close STF;
822  }
823 }
824 
825 sub update_status_file
826 {
827  $file = shift;
828  $mesg = shift;
829  open(STF,">$file");
830  print STF $mesg;
831  close STF;
832 }
833 
834 sub read_status_file
835 {
836  $file = shift;
837  open(STF,"<$file");
838  $status = <STF>;
839  chomp($status);
840  close STF;
841  return $status;
842 }
843 
844 sub execute_command
845 {
846  $cmd = shift;
847  print "Executing: $cmd \n";
848  my $ret = system("$cmd");
849  if ($ret != 0)
850  {
851  die "** Failed executing: $cmd \n";
852  }
853 }
854 
855 sub check_number_of_xml_and_pbs_files
856 {
857  $dirname = shift;
858  print "Searching for XML and PBS files in directory: $dirname \n";
859  opendir my $dir, $dirname or die "Can not open directory: $!";
860  my @xmlfiles = grep { /.xml$/ } readdir $dir;
861  rewinddir $dir;
862  my @pbsfiles = grep { /.pbs$/ } readdir $dir;
863  closedir $dir;
864 # print "XML files found: @xmlfiles \n";
865 # print "PBS files found: @pbsfiles \n";
866  $nxmlfiles = @xmlfiles;
867  $npbsfiles = @pbsfiles;
868  print "Found $nxmlfiles XML files and $npbsfiles PBS files. \n";
869  if($nxmlfiles != $npbsfiles)
870  {
871  return 1;
872  }
873  return 0;
874 }
Generated by   doxygen 1.8.11