submit_mcc Namespace Reference

Functions
def	call_project_py (xml, stg, act, test=False)
def	handle_stage (workflow_def, xml, workflow, stage, nocheck=False, test=False)

Variables
string	ECL_URL = "http://dbweb6.fnal.gov:8080/ECL/dunepro"
	now = datetime.now().strftime('%Y%m%d%H%M%S')
	log_stream = StringIO()
string	FORMAT = '%(asctime)s - submit_mcc.py - %(levelname)s - %(message)s'
	stream
	level
	INFO
	format
	parser = argparse.ArgumentParser(description='')
	groupArg = parser.add_argument_group('require arguments')
	metavar
	type
	help
	required
	action
	str
	default
	args = parser.parse_args()
	cwd = os.getcwd()
	ecl_entry
	name
	value
tuple	logfile = ('log_mcc_submit_{}.txt'.format(now))
	filename
	elconn
	response = elconn.post(ecl_entry)

Function Documentation

def submit_mcc.call_project_py	(		xml,
			stg,
			act,
			test = False
	)

Definition at line 26 of file submit_mcc.py.

def call_project_py(xml, stg, act, test=False):
    if test:
        tmpdir = "/pnfs/dune/scratch/dunepro/mcc10_test/test_{}".format(now)
        subprocess.call("mkdir -p {}".format(tmpdir), shell=True)
        overrides = ["wrap_proj", "-Onumevents=10",
                "-Ostage.numjobs=1", "-Ostage.outdir={tmp}",
                "-Ostage.workdir={tmp}".format(tmp=tmpdir)]
        logging.info("Submitting a test job with:")
        logging.info("                   numberjobs = 1;")
        logging.info("                   numevents  = 1;")
        logging.info("                   outdir  = {};".format(tmpdir))
        logging.info("                   workdir = {};".format(tmpdir))
    else:
        overrides = []
    prj_cmd = overrides + ["project.py", "--xml", xml, "--stage", stg,
        "--{}".format(act)]
    try:
        prj_out = subprocess.check_output(prj_cmd, stderr=subprocess.STDOUT)
        logging.info("project.py was run successfully.")
        logging.info("project.py output: {}".format(prj_out))
    except subprocess.CalledProcessError as e:
        logging.error("project.py exit with {}.".format(e.returncode))
        logging.error("project.py output: {}.".format(e.output))
        logging.error("Exiting now!")
        print(log_stream.getvalue())
        sys.exit(40)
    return

def submit_mcc.handle_stage	(		workflow_def,
			xml,
			workflow,
			stage,
			nocheck = False,
			test = False
	)

Definition at line 54 of file submit_mcc.py.

def handle_stage(workflow_def, xml, workflow, stage, nocheck=False, test=False):
    wf_dict = json.load(open(workflow_def))
    workflows = wf_dict["workflows"]
    if workflow not in workflows:
        logging.error("Workflow chosen is not defined in the json file.")
        print(log_stream.getvalue())
        sys.exit(30)
    if stage not in workflows[workflow]["stages"]:
        logging.error("Stage is not defined in the workflow.")
        print(log_stream.getvalue())
        sys.exit(31)
    stages = workflows[workflow]["stages"]
    actions = workflows[workflow]["actions"]
    stage_index = stages.index(stage)
    action = actions[stage_index]
    # not checking on previous stage if:
    #   1) '--no-check' is enabled;
    #   2) '--test' is enabled, cannot check on test jobs since xml file is 
    #       generated on the fly with parameter overrides;
    #   3) if the stage is the first stage of a workflow.
    if stage_index == 0 or test:
        nocheck=True
    if not nocheck:
        prev_stage = stages[stage_index-1]
        logging.info("Checking previous stage: {}".format(prev_stage))
        call_project_py(xml, prev_stage, "check")
        time.sleep(10)
    logging.info("Handle stage: {} with action: {}".format(stage, action))
    call_project_py(xml, stage, action, test)
    return

Variable Documentation

submit_mcc.action

Definition at line 96 of file submit_mcc.py.

submit_mcc.args = parser.parse_args()

Definition at line 116 of file submit_mcc.py.

submit_mcc.cwd = os.getcwd()

Definition at line 117 of file submit_mcc.py.

submit_mcc.default

Definition at line 100 of file submit_mcc.py.

submit_mcc.ecl_entry

Initial value:

= ECLEntry(
                category=args.ecl_category,
                tags=['MCCSubmission'],
                formname='MCC Submission',
                preformatted=False)

Definition at line 150 of file submit_mcc.py.

string submit_mcc.ECL_URL = "http://dbweb6.fnal.gov:8080/ECL/dunepro"

Definition at line 13 of file submit_mcc.py.

submit_mcc.elconn

Initial value:

= ECLConnection(url=ECL_URL, username=args.ecl_user,
                password=args.ecl_password)

Definition at line 173 of file submit_mcc.py.

submit_mcc.filename

Definition at line 170 of file submit_mcc.py.

string submit_mcc.FORMAT = '%(asctime)s - submit_mcc.py - %(levelname)s - %(message)s'

Definition at line 23 of file submit_mcc.py.

submit_mcc.format

Definition at line 24 of file submit_mcc.py.

submit_mcc.groupArg = parser.add_argument_group('require arguments')

Definition at line 87 of file submit_mcc.py.

submit_mcc.help

Definition at line 90 of file submit_mcc.py.

submit_mcc.INFO

Definition at line 24 of file submit_mcc.py.

submit_mcc.level

Definition at line 24 of file submit_mcc.py.

submit_mcc.log_stream = StringIO()

Definition at line 22 of file submit_mcc.py.

submit_mcc.logfile = ('log_mcc_submit_{}.txt'.format(now))

Definition at line 166 of file submit_mcc.py.

submit_mcc.metavar

Definition at line 89 of file submit_mcc.py.

submit_mcc.name

Definition at line 159 of file submit_mcc.py.

submit_mcc.now = datetime.now().strftime('%Y%m%d%H%M%S')

Definition at line 14 of file submit_mcc.py.

submit_mcc.parser = argparse.ArgumentParser(description='')

Definition at line 86 of file submit_mcc.py.

submit_mcc.required

Definition at line 90 of file submit_mcc.py.

submit_mcc.response = elconn.post(ecl_entry)

Definition at line 177 of file submit_mcc.py.

submit_mcc.str

Definition at line 100 of file submit_mcc.py.

submit_mcc.stream

Definition at line 24 of file submit_mcc.py.

submit_mcc.type

Definition at line 89 of file submit_mcc.py.

submit_mcc.value

Definition at line 159 of file submit_mcc.py.