submit_lar.sh
Go to the documentation of this file.
1 #! /bin/bash
2 #------------------------------------------------------------------
3 #
4 # Purpose: A general purpose larsoft batch submit script.
5 #
6 # Usage:
7 #
8 # submit_lar.sh [options]
9 #
10 # Lar options:
11 #
12 # -c, --config <arg> - Configuration (fcl) file (required).
13 # -s, --source <arg> - Input file (full path).
14 # -S, --source-list <arg> - Input file list (full path, one per line).
15 # -o, --output <arg> - Output file name.
16 # -n, --nevts <arg> - Number of events to process.
17 # --nskip <arg> - Number of events to skip.
18 # --nfile <arg> - Number of files to process (use with option -S).
19 # --nfile_skip <arg> - Number of files to skip (use with option -S).
20 # --args <args...> - Arguments for lar command line (place at end).
21 #
22 # Larsoft options.
23 #
24 # -r, --release <arg> - Release tag (default "development").
25 # -q, -b, --build <arg> - Release build qualifier (default "debug", or "prof").
26 # --localdir <arg> - Larsoft local test release directory (default none).
27 # --localtar <arg> - Tarball of local test release.
28 # --mrb - Use mrb-style environment initialization (see below).
29 # --srt - Use srt-style environment initialization (see below).
30 #
31 # Jobsub options.
32 #
33 # -N, --numjobs - Number of jobs to submit (default 1).
34 # --opportunistic - Use opportunistic grid.
35 # --OS <arg> - Specify OS (comma-separated list with no spaces, e.g. SL5,SL6).
36 #
37 # Microboone options.
38 #
39 # --ubfcl <arg> - Ubfcl version (default none).
40 #
41 # Other options.
42 #
43 # -h, --help - Print help.
44 # --group <arg> - Group or experiment (default $GROUP).
45 # --workdir <arg> - Work directory (default current directory).
46 # --outdir <arg> - Output directory.
47 # --cluster <arg> - Job cluster.
48 # --process <arg> - Process within cluster.
49 # --procmap <arg> - Name of process map file (override $PROCESS).
50 # --init-script <arg> - User initialization script execute.
51 # --init-source <arg> - User initialization script to source (bash).
52 # --end-script <arg> - User end-of-job script to execute.
53 #
54 # End options.
55 #
56 # MRB vs. SRT environment setup.
57 #
58 # Environmental setup inside the worker batch script is controlled by
59 # options --release (-r), --build (-b, -q), --localdir, and --localtar. These options are
60 # interpreted differently depending on whether one is using an MRB (--mrb)
61 # or SRT (--srt) environment. Options --srt and --mrb are mutually
62 # exclusive. If neither is specified, --mrb is the default.
63 #
64 # For SRT-style environment:
65 #
66 # a) Use option --release or -r to specify larsoft frozen release.
67 # b) Use option --build, -b, or -q to specify build type ("debug" or "prof")
68 # c) If you have a local test release, specify the (bluearc) location of
69 # the test release using --localdir, or use --localtar to specify the
70 # absolute or relative path of the local release tarball.
71 #
72 # For MRB-style environment:
73 #
74 # a) Use option --release or -r to specify uboonecode version. Note that
75 # larsoft is setup as a dependent product of uboonecode.
76 # b) Use option --build or -b to specify build full qualifiers (e.g.
77 # "debug:e4" or "e4:prof").
78 # c) Options --localdir and --localtar are used in a similar way as for
79 # SRT. Here, --localdir should point to, or your tarball should be
80 # made relative to, the mrb localProducts directory ($MRB_INSTALL).
81 # If your local test release includes uboonecode, then you do not
82 # need options --release or --build.
83 #
84 # Notes.
85 #
86 # 1. Each batch worker is uniquely identified by two numbers stored
87 # in environment variables $CLUSTER and $PROCESS (the latter is
88 # a small integer that starts from zero and varies for different
89 # jobs in a parallel job group). These environment variables are
90 # normally set by the batch system, but can be overridden by options
91 # --cluster and --process (e.g. to rerun failed jobs).
92 #
93 # It is not allowed to specify multiple jobs (-N option with >1 job)
94 # and to specify --process at the same time.
95 #
96 # 2. The work directory must be set to an existing directory owned
97 # by the submitter and readable by the batch worker. Files from the
98 # work directory are copied to the batch worker scratch directory at
99 # the start of the job.
100 #
101 # 3. The job configuration file (-c option), initialization and end-of-job
102 # scripts (optins --init-script, --init-source, --end-script) may
103 # be stored in the work directory specified by option --workdir, or they
104 # may be specified as absolute paths.
105 #
106 # 4. The output directory must exist and be writable by the batch
107 # worker (i.e. be group-writable for grid jobs). The worker
108 # makes a new subdirectory called ${CLUSTER}_${PROCESS} in the output
109 # directory and copies all files in the batch scratch directory there
110 # at the end of the job. If the output directory is not specified, the
111 # default is /grid/data/<group>/outstage/<user> (user is defined as
112 # owner of work directory).
113 #
114 # 5. The number of jobs option (-N or --numjobs) is used in two ways.
115 # It is passed to jobsub to generate the requested number of batch jobs,
116 # and it is passed to condor_lar.sh (as option --njobs) to control how
117 # condor_lar.sh delivers files and events to individual workers.
118 #
119 # Created: H. Greenlee, 29-Aug-2012
120 #
121 #------------------------------------------------------------------
122 
123 # Parse arguments.
124 
125 FCL=""
126 INFILE=""
127 INLIST=""
128 OUTFILE=""
129 NEVT=0
130 NSKIP=0
131 NFILE=0
132 NFILE_SKIP=0
133 ARGS=""
134 REL=""
135 QUAL=""
136 LOCALDIR=""
137 LOCALTAR=""
138 MRB=0
139 SRT=0
140 UBFCL=""
141 NUMJOBS=1
142 OPPORTUNISTIC=0
143 GRP=$GROUP
144 OS=""
145 WORKDIR=`pwd`
146 OUTDIR=""
147 CLUS=""
148 PROC=""
149 PROCMAP=""
150 INITSCRIPT=""
151 INITSOURCE=""
152 ENDSCRIPT=""
153 
154 while [ $# -gt 0 ]; do
155  case "$1" in
156 
157  # Help.
158  -h|--help )
159  awk '/^# Usage:/,/^# End options/{print $0}' $0 | cut -c3- | head -n -2
160  exit
161  ;;
162 
163  # Config file.
164  -c|--config )
165  if [ $# -gt 1 ]; then
166  FCL=$2
167  shift
168  fi
169  ;;
170 
171  # Input file.
172  -s|--source )
173  if [ $# -gt 1 ]; then
174  INFILE=$2
175  shift
176  fi
177  ;;
178 
179  # Input file list.
180  -S|--source-list )
181  if [ $# -gt 1 ]; then
182  INLIST=$2
183  shift
184  fi
185  ;;
186 
187  # Output file.
188  -o|--output )
189  if [ $# -gt 1 ]; then
190  OUTFILE=$2
191  shift
192  fi
193  ;;
194 
195  # Number of events.
196  -n|--nevts )
197  if [ $# -gt 1 ]; then
198  NEVT=$2
199  shift
200  fi
201  ;;
202 
203  # Number of events to skip.
204  --nskip )
205  if [ $# -gt 1 ]; then
206  NSKIP=$2
207  shift
208  fi
209  ;;
210 
211  # Number of files to process.
212  --nfile )
213  if [ $# -gt 1 ]; then
214  NFILE=$2
215  shift
216  fi
217  ;;
218 
219  # Number of files to skip.
220  --nfile_skip )
221  if [ $# -gt 1 ]; then
222  NFILE_SKIP=$2
223  shift
224  fi
225  ;;
226 
227  # General arguments for lar command line.
228  --args )
229  if [ $# -gt 1 ]; then
230  shift
231  ARGS=$@
232  break
233  fi
234  ;;
235 
236  # Release tag.
237  -r|--release )
238  if [ $# -gt 1 ]; then
239  REL=$2
240  shift
241  fi
242  ;;
243 
244  # Release build qualifier.
245  -q|-b|--build )
246  if [ $# -gt 1 ]; then
247  QUAL=$2
248  shift
249  fi
250  ;;
251 
252  # Local test release directory.
253  --localdir )
254  if [ $# -gt 1 ]; then
255  LOCALDIR=$2
256  shift
257  fi
258  ;;
259 
260  # Local test release tarball.
261  --localtar )
262  if [ $# -gt 1 ]; then
263  LOCALTAR=$2
264  shift
265  fi
266  ;;
267 
268  # MRB flag.
269  --mrb )
270  MRB=1
271  ;;
272 
273  # SRT flag.
274  --srt )
275  SRT=1
276  ;;
277 
278  # Ubfcl version.
279  --ubfcl )
280  if [ $# -gt 1 ]; then
281  UBFCL=$2
282  shift
283  fi
284  ;;
285 
286  # Number of batch jobs.
287  -N|--numjobs )
288  if [ $# -gt 1 ]; then
289  NUMJOBS=$2
290  shift
291  fi
292  ;;
293 
294  # Opportunistic grid flag.
295  --opportunistic )
296  OPPORTUNISTIC=1
297  ;;
298 
299  # Group.
300  -g|--group )
301  if [ $# -gt 1 ]; then
302  GRP=$2
303  shift
304  fi
305  ;;
306 
307  # OS.
308  --OS )
309  if [ $# -gt 1 ]; then
310  OS=$2
311  shift
312  fi
313  ;;
314 
315  # Work directory.
316  --workdir )
317  if [ $# -gt 1 ]; then
318  WORKDIR=$2
319  shift
320  fi
321  ;;
322 
323  # Output directory.
324  --outdir )
325  if [ $# -gt 1 ]; then
326  OUTDIR=$2
327  shift
328  fi
329  ;;
330 
331  # Job cluster.
332  --cluster )
333  if [ $# -gt 1 ]; then
334  CLUS=$2
335  shift
336  fi
337  ;;
338 
339  # Process within cluster.
340  --process )
341  if [ $# -gt 1 ]; then
342  PROC=$2
343  shift
344  fi
345  ;;
346 
347  # Process map.
348  --procmap )
349  if [ $# -gt 1 ]; then
350  PROCMAP=$2
351  shift
352  fi
353  ;;
354 
355  # User initialization script.
356  --init-script )
357  if [ $# -gt 1 ]; then
358  INITSCRIPT=$2
359  shift
360  fi
361  ;;
362 
363  # User source initialization script.
364  --init-source )
365  if [ $# -gt 1 ]; then
366  INITSOURCE=$2
367  shift
368  fi
369  ;;
370 
371  # User end-of-job script.
372  --end-script )
373  if [ $# -gt 1 ]; then
374  ENDSCRIPT=$2
375  shift
376  fi
377  ;;
378 
379  # Other.
380  * )
381  echo "Unknown option $1"
382  exit 1
383  esac
384  shift
385 done
386 
387 # Do MRB/SRT checks, and set some environment-specific defaults.
388 
389 # Make sure at lease one of MRB and SRT is specified.
390 # If neither is specified, set MRB by default.
391 
392 if [ $MRB -eq 0 -a $SRT -eq 0 ]; then
393  MRB=1
394 fi
395 
396 # Make sure both MRB and SRT are not set.
397 
398 if [ $MRB -ne 0 -a $SRT -ne 0 ]; then
399  echo "Both --mrb and --srt were specified."
400  exit 1
401 fi
402 
403 # Set defaults for MRB.
404 
405 if [ $MRB -ne 0 ]; then
406  if [ x$QUAL = x ]; then
407  QUAL="debug:e4"
408  fi
409 fi
410 
411 # Set defaults for SRT.
412 
413 if [ $SRT -ne 0 ]; then
414  if [ x$REL = x ]; then
415  REL="development"
416  fi
417  if [ x$QUAL = x ]; then
418  QUAL="debug"
419  fi
420 fi
421 
422 #echo "FCL=$FCL"
423 #echo "INFILE=$INFILE"
424 #echo "INLIST=$INLIST"
425 #echo "OUTFILE=$OUTFILE"
426 #echo "NEVT=$NEVT"
427 #echo "NSKIP=$NSKIP"
428 #echo "NFILE=$NFILE"
429 #echo "NFILE_SKIP=$NFILE_SKIP"
430 #echo "ARGS=$ARGS"
431 #echo "REL=$REL"
432 #echo "QUAL=$QUAL"
433 #echo "LOCALDIR=$LOCALDIR"
434 #echo "LOCALTAR=$LOCALTAR"
435 #echo "MRB=$MRB"
436 #echo "SRT=$SRT"
437 #echo "UBFCL=$UBFCL"
438 #echo "NUMJOBS=$NUMJOBS"
439 #echo "OPPORTUNISTIC=$OPPORTUNISTIC"
440 #echo "GRP=$GRP"
441 #echo "WORKDIR=$WORKDIR"
442 #echo "OUTDIR=$OUTDIR"
443 #echo "CLUS=$CLUS"
444 #echo "PROC=$PROC"
445 #echo "PROCMAP=$PROCMAP"
446 #echo "INITSCRIPT=$INITSCRIPT"
447 #echo "INITSOURCE=$INITSOURCE"
448 #echo "ENDSCRIPT=$ENDSCRIPT"
449 
450 # Done with arguments.
451 
452 # Make sure all required options have been specified.
453 
454 if [ x$GROUP = x ]; then
455  echo "Group not specified."
456  exit 1
457 fi
458 if [ x$OUTDIR = x ]; then
459  echo "Output directory not specified."
460  exit 1
461 fi
462 if [ x$WORKDIR = x ]; then
463  echo "Work directory not specified."
464  exit 1
465 fi
466 if [ x$FCL = x ]; then
467  echo "Configuration file not specified."
468  exit 1
469 fi
470 
471 # Construct options for jobsub
472 
473 JOBSUB_OPTS=''
474 if [ x$GRP != x ]; then
475  JOBSUB_OPTS="$JOBSUB_OPTS --group=$GRP"
476 fi
477 if [ $NUMJOBS -gt 1 ]; then
478  JOBSUB_OPTS="$JOBSUB_OPTS -N $NUMJOBS"
479 fi
480 JOBSUB_OPTS="$JOBSUB_OPTS --grid"
481 if [ $OPPORTUNISTIC -ne 0 ]; then
482  JOBSUB_OPTS="$JOBSUB_OPTS --opportunistic"
483 fi
484 if [ x$OS != x ]; then
485  JOBSUB_OPTS="$JOBSUB_OPTS --OS=$OS"
486 fi
487 
488 # Construct condor_lar.sh options.
489 
490 BATCH_OPTS=''
491 if [ x$FCL != x ]; then
492  BATCH_OPTS="$BATCH_OPTS -c $FCL"
493 fi
494 if [ x$INFILE != x ]; then
495  BATCH_OPTS="$BATCH_OPTS -s $INFILE"
496 fi
497 if [ x$INLIST != x ]; then
498  BATCH_OPTS="$BATCH_OPTS -S $INLIST"
499 fi
500 if [ x$OUTFILE != x ]; then
501  BATCH_OPTS="$BATCH_OPTS -o $OUTFILE"
502 fi
503 if [ $NEVT -gt 0 ]; then
504  BATCH_OPTS="$BATCH_OPTS -n $NEVT"
505 fi
506 if [ $NSKIP -gt 0 ]; then
507  BATCH_OPTS="$BATCH_OPTS --nskip $NSKIP"
508 fi
509 if [ $NFILE -gt 0 ]; then
510  BATCH_OPTS="$BATCH_OPTS --nfile $NFILE"
511 fi
512 if [ $NFILE_SKIP -gt 0 ]; then
513  BATCH_OPTS="$BATCH_OPTS --nfile_skip $NFILE_SKIP"
514 fi
515 if [ $NUMJOBS -gt 1 ]; then
516  BATCH_OPTS="$BATCH_OPTS --njobs $NUMJOBS"
517 fi
518 if [ x$REL != x ]; then
519  BATCH_OPTS="$BATCH_OPTS -r $REL"
520 fi
521 if [ x$QUAL != x ]; then
522  BATCH_OPTS="$BATCH_OPTS -q $QUAL"
523 fi
524 if [ x$LOCALDIR != x ]; then
525  BATCH_OPTS="$BATCH_OPTS --localdir $LOCALDIR"
526 fi
527 if [ x$LOCALTAR != x ]; then
528  BATCH_OPTS="$BATCH_OPTS --localtar $LOCALTAR"
529 fi
530 if [ $MRB -ne 0 ]; then
531  BATCH_OPTS="$BATCH_OPTS --mrb"
532 fi
533 if [ $SRT -ne 0 ]; then
534  BATCH_OPTS="$BATCH_OPTS --srt"
535 fi
536 if [ x$UBFCL != x ]; then
537  BATCH_OPTS="$BATCH_OPTS --ubfcl $UBFCL"
538 fi
539 if [ x$GRP != x ]; then
540  BATCH_OPTS="$BATCH_OPTS --group $GRP"
541 fi
542 if [ x$WORKDIR != x ]; then
543  BATCH_OPTS="$BATCH_OPTS --workdir $WORKDIR"
544 fi
545 if [ x$OUTDIR != x ]; then
546  BATCH_OPTS="$BATCH_OPTS --outdir $OUTDIR"
547 fi
548 if [ x$CLUS != x ]; then
549  BATCH_OPTS="$BATCH_OPTS --cluster $CLUS"
550 fi
551 if [ x$PROC != x ]; then
552  BATCH_OPTS="$BATCH_OPTS --process $PROC"
553 fi
554 if [ x$PROCMAP != x ]; then
555  BATCH_OPTS="$BATCH_OPTS --procmap $PROCMAP"
556 fi
557 if [ x$INITSCRIPT != x ]; then
558  BATCH_OPTS="$BATCH_OPTS --init-script $INITSCRIPT"
559 fi
560 if [ x$INITSOURCE != x ]; then
561  BATCH_OPTS="$BATCH_OPTS --init-source $INITSOURCE"
562 fi
563 if [ x$ENDSCRIPT != x ]; then
564  BATCH_OPTS="$BATCH_OPTS --end-script $ENDSCRIPT"
565 fi
566 
567 if [ -n "$ARGS" ]; then
568  BATCH_OPTS="$BATCH_OPTS $ARGS"
569 fi
570 
571 # Make a fresh copy of batch worker script in work directory and go there.
572 
573 cp `which condor_lar.sh` $WORKDIR
574 cd $WORKDIR
575 
576 # Submit jobs.
577 
578 CMD="jobsub $JOBSUB_OPTS condor_lar.sh $BATCH_OPTS"
579 echo $CMD
580 eval $CMD
Generated by   doxygen 1.8.11