submit_flux.py

Go to the documentation of this file.

##################################################
##
## submit_flux.py
##
## Laura Fields
## Aug 2012
##
## Submits g4lbne flux jobs using macro, input, number of
## protons, number of jobs and output directory
## specified by user
##
###################################################

from optparse import OptionParser
import sys, os, subprocess, datetime

###################################################
##
## Helper Functions
##
###################################################
def remove_file(myfile):
        print "REMOVING",myfile
#    if myfile.startswith("/pnfs/"):
        os.system("ifdh rm "+myfile)



###################################################
#
# Determine default g4lbne directory
# (the directory this script is in)
#
###################################################
scriptdir = os.path.abspath(sys.argv[0]+"/../..")
print "default g4lbne directory:",scriptdir

###################################################
#
# Setup parser that reads in options
#
###################################################

usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)

parser.add_option("-g", "--g4lbne_dir", dest="g4lbne_dir",
                  help="g4lbne directory", default=scriptdir)
parser.add_option("-p", "--physics_list", dest="physics_list",
                  help="Geant4 Physics List", default="QGSP_BERT");
parser.add_option("-m", "--macro", dest="macro",
                  help="Template Macro", default="Nominal")
parser.add_option("-d", "--mode", dest="mode",
                  help="neutrino or antineutrino", default="neutrino")
parser.add_option("-n", "--n_events", dest="n_events",
                  help="Number of events per job", default=100000)
parser.add_option("-f", "--first_job", dest="first_job",
                  help="First job number", default="1")
parser.add_option("-l", "--last_job", dest="last_job",
                  help="Last job number", default="1")
parser.add_option("-o", "--output_dir", dest="output_dir",
                  help="Output directory", default="/pnfs/dune/scratch/users/$USER/fluxfiles/g4lbne")
parser.add_option("-a", "--macro_dir", dest="macro_dir",
                  help="Macro directory", default="macros")
parser.add_option("-v", "--interactive", dest="interactive",action="store_true",
                  help="Run interactively", default=False)
parser.add_option("-s","--osg",dest="osg",action="store_true",help="Option to run on OSG.  Does not work yet", default=False) 
parser.add_option("-t", "--notuples", dest="notuples",action="store_true",help="Don't write ntuple output",default=False)
parser.add_option("-e","--test",dest="xtest",action="store_true",help="Option for running a test job.  Runs a little quicker",default=False)
parser.add_option("-P","--ppfx_options",dest="ppfx_options",action="store_true",
                  help="Create ppfx ntuples parallely with g4lbnf",default=False)
parser.add_option("-M","--MODE",dest="MODE",help="specify the setup mode for ppfx (REF or OPT)",default="NA")

(options, args) = parser.parse_args()

# Expand environment variables if necessary
if options.g4lbne_dir.find("$USER")>=0:
    options.g4lbne_dir = options.g4lbne_dir.replace("$USER",os.getenv("USER"));
if options.output_dir.find("$USER")>=0:
    options.output_dir = options.output_dir.replace("$USER",os.getenv("USER"));

###################################################
#
# Determine G4LBNE Version
#
###################################################
version="v3r5p5"

###################################################
#
# Check validity of specified options
#
###################################################
g4lbne_executable=options.g4lbne_dir+"/g4lbnf";
ppfx_dir = options.g4lbne_dir+"/ppfx/";
ppfx_executable = ppfx_dir+"bin/doReweight_dk2nu_dune"
if not os.path.exists(g4lbne_executable):
    print "ERROR: I can't find a g4lbne executable at "+g4lbne_executable
    sys.exit()
    
ppfx_setup = options.g4lbne_dir+"/ppfx/setup_for_dune_cvmfs.sh "+options.MODE
ppfx_xml = options.g4lbne_dir+"/ppfx/scripts/inputs_default.xml"
if options.ppfx_options:
        if not os.path.exists(ppfx_xml):
                print "ERROR: I cannot find the ppfx option file at "+ppfx_xml
                sys.exit()
                if not os.path.exists(ppfx_setup):
                        print "ERROR: I cannot find ppfx setup script at "+ppfx_setup
                        sys.exit()

if options.ppfx_options:
        if not os.path.exists(options.g4lbne_dir+"/ppfx/setup_for_dune_cvmfs.sh"):
                print "I can't find a ppfx setup at "+ppfx_setup
                sys.exit()
        mode=str(options.MODE)
        print mode
        if mode=="REF":
                print "Reference mode for ppfx"
        elif mode=="OPT":
                print "Optimized mode for ppfx"
        else:
                print "Specify REF or OPT mode to execute ppfx setup script (option -M)"
                sys.exit()


g4lbne_setup=options.g4lbne_dir+"/setups/setup_g4lbne_cvmfs.sh"
if not os.path.exists(g4lbne_setup):
    print "ERROR: I can't find a g4lbne setup script at "+g4lbne_setup
    sys.exit()
    
macro_template = options.macro_dir+"/"+options.macro+".mac"
if options.macro.startswith("Optimizations"):
    splitmacro = options.macro.split("-")
    print splitmacro
    macro_template = options.macro_dir+"/Optimizations/"+splitmacro[1]+"/"+splitmacro[1]+"_"+splitmacro[2]+".mac"
if not os.path.exists(macro_template):
    print "ERROR: I can't find a macro at "+macro_template
    sys.exit()

histomaker_macro=options.g4lbne_dir+"/ProductionScripts/makeFluxHistogramsL";
if not os.path.exists(histomaker_macro):
    print "ERROR: I can't find a histogram macro at "+histomaker_macro
    sys.exit()

physics_list = options.physics_list
print "Using GEANT4 physics list",physics_list

###################################################
#
# Print options
#
###################################################
if options.interactive: 
    print "Running",str(int(options.last_job)-int(options.first_job)+1),"jobs starting with JobID",options.first_job,"."
    print "Submitting",str(int(options.last_job)-int(options.first_job)+1),"jobs starting with JobID",options.first_job,"."
print "Using the g4lbne installed at",options.g4lbne_dir
print "Each job will have",options.n_events,"protons on target."
print "Using macro template:",macro_template
print "Output will be written to:",options.output_dir
if options.notuples:
    print "Will only write histogram output -- no ntuples"

###################################################
#
# Create Directories
#
###################################################

output_macro_dir = "/pnfs/dune/scratch/users/"+os.getenv("USER")+"/fluxfiles/g4lbne/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/macros/"
output_wrapper_dir = "/pnfs/dune/scratch/users/"+os.getenv("USER")+"/fluxfiles/g4lbne/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/wrappers/"
output_executable_dir = "/pnfs/dune/scratch/users/"+os.getenv("USER")+"/fluxfiles/g4lbne/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/executables/"
flux_dir = options.output_dir+"/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/flux/"
log_dir = options.output_dir+"/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/logfiles/"
ppfx_flux_dir = options.output_dir+"/"+version+"/"+physics_list+"/"+options.macro+"/"+str(options.mode)+"/ppfx/"
ppfx_hadron_dir = ppfx_flux_dir+"hadron/flux/"
ppfx_hadron_log = ppfx_flux_dir+"hadron/logfiles/"
ppfx_imap_dir = ppfx_flux_dir+"imap/flux/"
ppfx_imap_log = ppfx_flux_dir+"imap/logfiles/"
print "Creating",output_macro_dir
if not os.path.exists(output_macro_dir):
    os.makedirs(output_macro_dir)
print "Creating",output_wrapper_dir
if not os.path.exists(output_wrapper_dir):
    os.makedirs(output_wrapper_dir)
print "Creating",output_executable_dir
if not os.path.exists(output_executable_dir):
    os.makedirs(output_executable_dir)
print "Creating",flux_dir
if not os.path.exists(flux_dir):
    os.makedirs(flux_dir)
if not os.path.exists(flux_dir+"/histos/"):
    os.makedirs(flux_dir+"/histos/")
print "Creating",log_dir
if not os.path.exists(log_dir):
    os.makedirs(log_dir)
if options.ppfx_options:
        print "Creating",ppfx_hadron_dir
        if not os.path.exists(ppfx_hadron_dir):
            os.makedirs(ppfx_hadron_dir)
        print "Creating",ppfx_hadron_log
        if not os.path.exists(ppfx_hadron_log):
            os.makedirs(ppfx_hadron_log)
###########################################
#
#Uncomment the following lines if you want interaction map from ppfx
#
############################################
"""     print "Creating",ppfx_imap_dir
        if not os.path.exists(ppfx_imap_dir):
            os.makedirs(ppfx_imap_dir)
        print "Creating",ppfx_imap_log
        if not os.path.exists(ppfx_imap_log):
            os.makedirs(ppfx_imap_log)
"""

if not os.path.exists(output_macro_dir):
    print "ERROR: could not create macro directory: "+output_macro_dir
    sys.exit()

if not os.path.exists(output_wrapper_dir):
    print "ERROR: could not create macro directory: "+output_wrapper_dir
    sys.exit()

if not os.path.exists(output_executable_dir):
    print "ERROR: could not create executable directory: "+output_executable_dir
    sys.exit()

if not os.path.exists(flux_dir):
    print "ERROR: could not create flux directory: "+flux_dir
    sys.exit()

if not os.path.exists(log_dir):
    print "ERROR: could not create log directory: "+log_dir
    sys.exit()
if options.ppfx_options:
    if not os.path.exists(ppfx_hadron_dir):
        print "ERROR: could not create macro directory: "+ppfx_hadron_dir
        sys.exit()
    if not os.path.exists(ppfx_hadron_log):
        print "ERROR: could not create macro directory: "+ppfx_hadron_log
        sys.exit()

###########################################
#
#Uncomment the following lines if you want interaction map from ppfx
#
############################################        
"""     if not os.path.exists(ppfx_imap_flux):
                print "ERROR: could not create flux directory: "+ppfx_imap_flux
        sys.exit()

        if not os.path.exists(ppfx_imap_log):
            print "ERROR: could not create log directory: "+ppfx_imap_log
                sys.exit()"""
os.chmod(output_macro_dir,0777)
os.chmod(output_executable_dir,0777)
os.chmod(output_wrapper_dir,0777)
os.chmod(flux_dir,0777)
os.chmod(flux_dir+"/histos/",0777)
os.chmod(log_dir,0777)

temp_dir = " "
if not options.interactive:
    temp_dir = "$_CONDOR_SCRATCH_DIR/"

###################################################
#
# Make a tar file of the user's executable, setup area
#
###################################################
cwd = os.getcwd()
os.chdir(options.g4lbne_dir)
if not options.ppfx_options:
        os.system("tar -czf local_install.tar.gz ProductionScripts CMakeFiles shared_libraries g4lbnf build setups g4lbnfCint_rdict.pcm include locations")
else:
        #and we assume that we also need ppfx..assumes that ppfx is installed in ppfx directory 
        os.system("tar -czf local_install.tar.gz ProductionScripts CMakeFiles shared_libraries g4lbnf build setups g4lbnfCint_rdict.pcm include locations ppfx" )

executable_file = output_executable_dir+"g4lbne_"+version+"_"+physics_list+"_"+options.macro+"_"+str(options.mode)+".tar.gz"
if os.path.exists(executable_file):
        os.remove(executable_file)
os.system("cp local_install.tar.gz "+executable_file)
os.remove("local_install.tar.gz")


###################################################
#
# Loop over requested jobs
#
###################################################
file_prefix = "g4lbne_"+version+"_"+physics_list+"_"+options.macro+"_"+str(options.mode)
for i in range(int(options.first_job),int(options.last_job)+1):

    if 1>0:
        # Write macro file
        oldmac = open(macro_template)    
        new_macro_filename = output_macro_dir+os.path.basename(macro_template).replace(".mac","_"+str(i).zfill(3)+".mac")

        if os.path.exists(new_macro_filename):
                os.remove(new_macro_filename)

        newmac = open(new_macro_filename, 'w')
        horn_current_set = False
        for s in oldmac.xreadlines():
            if s.find("/LBNE/output/OutputNtpFileName")>=0:
                newmac.write("/LBNE/output/OutputNtpFileName "+temp_dir+file_prefix+"\n")
            elif s.find("/LBNE/output/OutputDk2NuFileName")>=0:
                newmac.write("/LBNE/output/OutputDk2NuFileName "+temp_dir+file_prefix+"\n")
            elif s.find("/LBNE/rndm/setRndmSeed")>=0:
                newmac.write("/LBNE/rndm/setRndmSeed "+str(i)+"\n")
            elif s.find("/LBNE/run/setRunID")>=0:
                newmac.write("/LBNE/run/setRunID "+str(i)+"\n")
            elif s.find("/LBNE/run/NEvents")>=0:
                newmac.write("/LBNE/run/NEvents "+str(options.n_events)+"\n")
            elif s.find("/LBNE/det/seHornCurrent")>=0:
                    splitline = s.split()
                    if options.mode=="antineutrino" and not splitline[1].strip().startswith("-"):
                        newmac.write("/LBNE/det/seHornCurrent "+"-"+splitline[1]+" \n")
                    elif options.mode=="neutrino" and splitline[1].startswith("-"):
                        newmac.write("/LBNE/det/seHornCurrent "+splitline[1][1:]+" \n")
                    else:
                        newmac.write(s)
                    horn_current_set = True
            else:
                newmac.write(s);

        if not horn_current_set:
            print "ERROR: The macro "+macro_template+" does set the horn current; please add a line such as /LBNE/det/seHornCurrent 200 kA to your macro template"
            sys.exit()

        newmac.close()

# Write the script that will run on the grid
if 1>0:

    script_filename = output_wrapper_dir+os.path.basename(macro_template).replace(".mac",".sh")
    
    if os.path.exists(script_filename):
            os.remove(script_filename)

            

    wrapper = open(script_filename, 'w')
    on_grid = "true";
    if(options.interactive):
        on_grid = "false";
        wrapper.write("source "+g4lbne_setup+" \n\n")
                
    else:
            local_g4lbne_setup = "setups/"+os.path.basename(g4lbne_setup)
            local_ppfx_setup = "ppfx/"+os.path.basename(ppfx_setup)
            local_tar_path = "$_CONDOR_SCRATCH_DIR/"+os.path.basename(executable_file)
            wrapper.write("source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups \n\n")

            wrapper.write("setup ifdhc \n\n")
            wrapper.write("ifdh cp "+ executable_file+" "+local_tar_path+" \n\n")
            wrapper.write("tar -xvzf "+ local_tar_path+" \n\n")

            wrapper.write("source "+local_g4lbne_setup+" \n\n")

            if options.xtest:
                    wrapper.write('os.environ["IFDH_GRIDFTP_EXTRA"]="-st 10" \n\n')   # waits only 10 seconds 
                    wrapper.write('os.environ["IFDH_CP_MAXRETRIES"]="2" \n\n')


    offset = options.first_job
    wrapper.write("export run_number=$(($PROCESS+"+offset+")) \n")
    wrapper.write("export padded_run_number=`printf \"%03g\" $(($PROCESS+"+offset+"))` \n")
    wrapper.write("export padded_run_number5=`printf \"%05g\" $(($PROCESS+"+offset+"))` \n")
    wrapper.write("\nenv\n\n")
    new_macro_filename = output_macro_dir+os.path.basename(macro_template).replace(".mac","_${padded_run_number}.mac")
    print output_macro_dir
    #sys.exit()
    local_macro_path = os.path.basename(new_macro_filename)

    local_g4lbne_executable = "g4lbnf"
    local_ppfx_executable = "ppfx/bin/doReweight_dk2nu_dune"
    local_ppfx_xml = "ppfx/scripts/inputs_default.xml"

    local_histomaker_macro = "ProductionScripts/makeFluxHistogramsL";

    if options.interactive:
            wrapper.write(g4lbne_executable+" --physicslist "+physics_list+" "+new_macro_filename+'\n\n')
            if options.ppfx_options:
                    dk2nu_output = file_prefix+"_${padded_run_number5}.dk2nu.root";
                    ppfx_ndhisto_output = "histos_"+file_prefix+"_LBNFND_ppfx_${padded_run_number5}.root";
                    wrapper.write("source "+ppfx_setup+"\n\n")
                    wrapper.write("echo $PWD") #need to figure out where i am usually
                    wrapper.write(ppfx_executable+" "+flux_dir+dk2nu_output+" "+ppfx_hadron_dir+ppfx_ndhisto_output+" "+ppfx_xml+"\n\n")
    else:
            wrapper.write("ifdh cp "+ new_macro_filename+" "+local_macro_path+" \n\n")
            wrapper.write("if [ -f "+local_macro_path+" ];\n")
            wrapper.write("then\n")
            wrapper.write("   "+local_g4lbne_executable+" --physicslist "+physics_list+" "+local_macro_path+'\n')
            wrapper.write("else\n")
            wrapper.write('   echo "'+local_macro_path+' does not exit.  Exiting."\n')
            wrapper.write("   exit 1 \n")
            wrapper.write("fi \n\n")
                   




    # copy the output
    ntuple_output = file_prefix+"_${padded_run_number}.root";
    dk2nu_output = file_prefix+"_${padded_run_number5}.dk2nu.root";
    ndhisto_output = "histos_"+file_prefix+"_${run_number}_LBNEND.root";
    fdhisto_output = "histos_"+file_prefix+"_${run_number}_LBNEFD.root";
    ndfmchisto_output = "histos_"+file_prefix+"_${run_number}_LBNEND_fastmc.root";
    fdfmchisto_output = "histos_"+file_prefix+"_${run_number}_LBNEFD_fastmc.root";
    ndglobeshisto_output = "histos_"+file_prefix+"_${run_number}_LBNEND_globes_flux.txt";
    fdglobeshisto_output = "histos_"+file_prefix+"_${run_number}_LBNEFD_globes_flux.txt";
    ppfx_ndhisto_output = "histos_"+file_prefix+"_LBNFND_ppfx_${padded_run_number5}.root";
    ppfx_fdhisto_output = "histos_"+file_prefix+"_LBNFFD_ppfx_${padded_run_number5}.root";
    
    if not options.notuples:
            wrapper.write("if [ -f "+temp_dir+ntuple_output+" ];\n")
            wrapper.write("then\n")
            wrapper.write("   ifdh cp  "+ temp_dir+ntuple_output+" "+flux_dir+ntuple_output+"\n")
            wrapper.write("fi \n")

            wrapper.write("if [ -f "+temp_dir+dk2nu_output+" ];\n")
            wrapper.write("then\n")
            wrapper.write("   ifdh cp "+ temp_dir+dk2nu_output+" "+flux_dir+dk2nu_output+"\n")
            wrapper.write("fi \n")



    # make histograms
    wrapper.write(local_histomaker_macro+' --user='+os.getenv("USER")+' --version='+version+' --macro='+options.macro+' --current='+str(options.mode)+' --location=LBNEND --physics_list='+physics_list+' --n_files=1 --start_index=${run_number} --pot_per_file='+str(options.n_events)+' --on_grid='+on_grid+'\n')
    wrapper.write(local_histomaker_macro+' --user='+os.getenv("USER")+' --version='+version+' --macro='+options.macro+' --current='+str(options.mode)+' --location=LBNEFD --physics_list='+physics_list+' --n_files=1 --start_index=${run_number} --pot_per_file='+str(options.n_events)+' --on_grid='+on_grid+'\n')
    if options.ppfx_options:
            wrapper.write("source "+local_ppfx_setup)
            wrapper.write("  "+local_ppfx_executable+" "+temp_dir+dk2nu_output+" "+temp_dir+ppfx_ndhisto_output+" "+local_ppfx_xml+"\n")
            wrapper.write("  "+local_ppfx_executable+" "+temp_dir+dk2nu_output+" "+temp_dir+ppfx_fdhisto_output+" "+local_ppfx_xml+"\n")
            
                          

    wrapper.write("ls -lrt $_CONDOR_SCRATCH_DIR\n")

    outfiles = temp_dir+ndhisto_output+" "+temp_dir+fdhisto_output+" "+temp_dir+ndfmchisto_output+" "+temp_dir+fdfmchisto_output+" "+temp_dir+ndglobeshisto_output+" "+temp_dir+fdglobeshisto_output+" "

    wrapper.write("if [ -f "+temp_dir+ndhisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+ndhisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")

    wrapper.write("if [ -f "+temp_dir+fdhisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+fdhisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")

    wrapper.write("if [ -f "+temp_dir+ndfmchisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+ndfmchisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")

    wrapper.write("if [ -f "+temp_dir+fdfmchisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+fdfmchisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")

    wrapper.write("if [ -f "+temp_dir+ndglobeshisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+ndglobeshisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")

    wrapper.write("if [ -f "+temp_dir+fdglobeshisto_output+" ];\n")
    wrapper.write("then\n")
    wrapper.write("   ifdh cp -D "+ temp_dir+fdglobeshisto_output+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("fi \n")
    if options.ppfx_options:          
            wrapper.write("if [ -f "+temp_dir+ppfx_ndhisto_output+" ];\n")
            wrapper.write("then\n")
            wrapper.write("  ifdh cp -D "+ temp_dir+ppfx_ndhisto_output+" "+ppfx_hadron_dir+"\n")
            wrapper.write("fi \n")
            wrapper.write("if [ -f "+temp_dir+ppfx_fdhisto_output+" ];\n")
            wrapper.write("then\n")
            wrapper.write("  ifdh cp -D "+temp_dir+ppfx_fdhisto_output+" "+ppfx_hadron_dir+"\n")
            wrapper.write("fi\n")

    #wrapper.write("ifdh cp -D "+outfiles+" "+flux_dir+"/histos/"+"\n")
    wrapper.write("echo 'ALL DONE'\n")
    wrapper.close();
    
    os.chmod(script_filename,0777);
    
    # remove old files to prevent permissions conflicts
    logfile = log_dir + ntuple_output.split("/")[-1].replace(".root",".log").replace("_${padded_run_number}","_\$PROCESS");
    if os.path.exists(logfile):
        os.remove(logfile)
    if os.path.exists(ntuple_output):
        os.remove(ntuple_output);
    if os.path.exists(dk2nu_output):
        os.remove(dk2nu_output);
    if os.path.exists(ndhisto_output):
        os.remove(ndhisto_output);
    if os.path.exists(fdhisto_output):
        os.remove(fdhisto_output);
    if os.path.exists(ndfmchisto_output):
        os.remove(ndfmchisto_output);
        
    if os.path.exists(fdfmchisto_output):
        os.remove(fdfmchisto_output);
    if os.path.exists(ndglobeshisto_output):
        os.remove(ndglobeshisto_output);
    if os.path.exists(fdglobeshisto_output):
        os.remove(fdglobeshisto_output);

        
        
    print "logfile",logfile
    # Submit the job
    if options.interactive:
            print "type "+script_filename+" to execute"
    elif options.osg:
#           os.system("jobsub_submit.py -N "+str(int(options.last_job)-int(options.first_job)+1)+" --resource-provides=usage_model=OFFSITE --OS=SL6 --disk=9000MB --site=SU-OG --group=dune -L "+logfile+" file:///"+script_filename);
            os.system("jobsub_submit.py -N "+str(int(options.last_job)-int(options.first_job)+1)+" --resource-provides=usage_model=OFFSITE --OS=SL6 --disk=9000MB --site=Nebraska,Wisconsin,FNAL,MWT2,Clemson,Cornell --expected-lifetime 4h --group=dune -L "+logfile+" file:///"+script_filename);
#           os.system("jobsub_submit.py --resource-provides=usage_model=OFFSITE --OS=SL6 --disk=9000MB --site=Nebraska,Caltech,Wisconsin,MWT2 --group=dune -L "+logfile+" file:///"+script_filename);
#           os.system("jobsub_submit.py --resource-provides=usage_model=OFFSITE --OS=SL6 --disk=9000MB --site=FNAL --group=dune -L "+logfile+" file:///"+script_filename);
    else:
            jobsub_command = "jobsub_submit.py -N "+str(int(options.last_job)-int(options.first_job)+1)+" --resource-provides=usage_model=OPPORTUNISTIC --expected-lifetime 4h --OS=SL6 --group=dune -L "+logfile+" file://"+script_filename;
            if options.xtest:
                    jobsub_command = "jobsub_submit.py -N "+str(int(options.last_job)-int(options.first_job)+1)+" --resource-provides=usage_model=OPPORTUNISTIC --expected-lifetime 1h --memory 500MB --OS=SL6 --group=dune -L "+logfile+" file://"+script_filename;
            print jobsub_command
            os.system(jobsub_command)


    print "DONE"