util::TFileMetadataDUNE Class Reference

#include <TFileMetadataDUNE.h>

Classes
struct	metadata

Public Member Functions
	TFileMetadataDUNE (fhicl::ParameterSet const &pset, art::ActivityRegistry &reg)
	~TFileMetadataDUNE ()
void	reconfigure (fhicl::ParameterSet const &p)

Public Attributes
metadata	md
std::set< art::SubRunID >	fSubRunNumbers
std::map< std::string, std::string >	mdmap

Private Member Functions
void	postBeginJob ()
void	postOpenFile (std::string const &fn)
void	postEvent (art::Event const &ev, art::ScheduleContext)
void	postCloseOutput (art::OutputFileInfo const &)
void	postBeginSubRun (art::SubRun const &subrun)
void	postEndJob ()

Private Attributes
bool	fGenerateTFileMetadata
std::string	frunType
std::string	fJSONFileName

Detailed Description

Definition at line 30 of file TFileMetadataDUNE.h.

Constructor & Destructor Documentation

util::TFileMetadataDUNE::TFileMetadataDUNE	(	fhicl::ParameterSet const &	pset,
		art::ActivityRegistry &	reg
	)

Definition at line 73 of file TFileMetadataDUNE_service.cc.

                                                                    :
  fGenerateTFileMetadata(false)                                                 
{
  reconfigure(pset);

  // Register for callbacks.
 
  reg.sPostBeginJob.watch(this, &TFileMetadataDUNE::postBeginJob);
  reg.sPostOpenFile.watch(this, &TFileMetadataDUNE::postOpenFile);
  reg.sPostEndJob.watch(this, &TFileMetadataDUNE::postEndJob);
  reg.sPostProcessEvent.watch(this, &TFileMetadataDUNE::postEvent);
  reg.sPostCloseOutputFile.watch(this, &TFileMetadataDUNE::postCloseOutput);
  reg.sPostBeginSubRun.watch(this, &TFileMetadataDUNE::postBeginSubRun);
}

util::TFileMetadataDUNE::~TFileMetadataDUNE

(

)

Definition at line 91 of file TFileMetadataDUNE_service.cc.

{
}

Member Function Documentation

void util::TFileMetadataDUNE::postBeginJob ( )

private

Definition at line 112 of file TFileMetadataDUNE_service.cc.

{ 
  // only generate metadata when this is true
  if (!fGenerateTFileMetadata) return;
    
  // get the start time  
  md.fstart_time = time(0); 
  
  // Get art metadata service and extract paramters from there
  art::ServiceHandle<art::FileCatalogMetadata> artmds;
  
  art::FileCatalogMetadata::collection_type artmd;
  artmds->getMetadata(artmd);
  
  for(auto const & d : artmd) {
    mdmap[d.first] = d.second;
    //std::cout << d.first << " " << d.second << std::endl;
  }
    
  std::map<std::string,std::string>::iterator it;
  
  // if a certain paramter/key is not found, assign an empty string value to it
  
  if ((it=mdmap.find("application.family"))!=mdmap.end()) {
    std::cout << "Setting applicationFamily: " << it->second << std::endl;
    std::get<0>(md.fapplication) = it->second;
  }
  else {
    std::cout << "Setting applicationFamily: empty" << std::endl;
    std::get<0>(md.fapplication) = "\" \"";   
  }
   
  if ((it=mdmap.find("art.process_name"))!=mdmap.end()) {
    std::cout << "Setting process_name: " << it->second << std::endl;
    std::get<1>(md.fapplication) = it->second;
  }
  else {
    std::cout << "Setting process_name: empty" << std::endl;
    std::get<1>(md.fapplication) = "\" \"";  
  }
  
  if ((it=mdmap.find("application.version"))!=mdmap.end()) std::get<2>(md.fapplication) = it->second;
  else  std::get<2>(md.fapplication) = "\" \"";   

  if ((it=mdmap.find("application.version"))!=mdmap.end()) std::get<2>(md.fapplication) = it->second;
  else  std::get<2>(md.fapplication) = "\" \"";   
  
  if ((it=mdmap.find("group"))!=mdmap.end()) md.fgroup = it->second;
  else md.fgroup = "\" \"";   
    
  if ((it=mdmap.find("file_type"))!=mdmap.end()) md.ffile_type = it->second;
  else  md.ffile_type = "\" \"";  
    
  if ((it=mdmap.find("run_type"))!=mdmap.end()) frunType = it->second;
  else frunType = "\" \"";                      

  //std::cout << "Run type: " << frunType << std::endl;

  //Remove this
  if ((it=mdmap.find("art.run_type"))!=mdmap.end()) frunType = it->second;
  else frunType = "\" \"";                      
  //std::cout << "Run type: " << frunType << std::endl;

}

void util::TFileMetadataDUNE::postBeginSubRun ( art::SubRun const &  subrun )

private

Definition at line 240 of file TFileMetadataDUNE_service.cc.

{

  if(!fGenerateTFileMetadata) return;

  art::RunNumber_t run = sr.run();
  art::SubRunNumber_t subrun = sr.subRun();
  art::SubRunID srid = sr.id();

  // save run, subrun and runType information once every subrun
  if (fSubRunNumbers.count(srid) == 0){
    fSubRunNumbers.insert(srid);
    md.fruns.push_back(make_tuple(run, subrun, frunType));
  }
}

void util::TFileMetadataDUNE::postCloseOutput ( art::OutputFileInfo const &  output_file )

private

Definition at line 223 of file TFileMetadataDUNE_service.cc.

{
 
  if(!fGenerateTFileMetadata) return;   
  
  
  // save the first event
  //if (md.fevent_count == 0) md.ffirst_event = event;
  //md.flast_event = event;
  // event counter
  md.fFileName = output_file.fileName();
    
}

void util::TFileMetadataDUNE::postEndJob ( )

private

Definition at line 258 of file TFileMetadataDUNE_service.cc.

{
        
  // Do nothing if generating TFile metadata is disabled.       
  if(!fGenerateTFileMetadata) return;   
  
  // get metadata from the FileCatalogMetadataDUNE service, which is filled on its construction
        
  art::ServiceHandle<util::FileCatalogMetadataDUNE> paramhandle; 
  md.fMCGenerators =            paramhandle->MCGenerators();                      
  md.fMCOscillationP =          paramhandle->MCOscillationP();         
  md.fMCTriggerListVersion =    paramhandle->MCTriggerListVersion();              
  md.fMCBeamEnergy =            paramhandle->MCBeamEnergy();                      
  md.fMCBeamFluxID =            paramhandle->MCBeamFluxID();                      
  md.fMCName =                  paramhandle->MCName();                  
  md.fMCDetectorType =          paramhandle->MCDetectorType();         
  md.fMCNeutrinoFlavors =       paramhandle->MCNeutrinoFlavors();       
  md.fMCMassHierarchy =         paramhandle->MCMassHierarchy();        
  md.fMCMiscellaneous =         paramhandle->MCMiscellaneous();        
  md.fMCGeometryVersion =       paramhandle->MCGeometryVersion();       
  md.fMCOverlay =               paramhandle->MCOverlay();               
  md.fDataRunMode =             paramhandle->DataRunMode();                       
  md.fDataDetectorType =        paramhandle->DataDetectorType();        
  md.fDataName =                paramhandle->DataName();                
  md.fStageName =               paramhandle->StageName();              

  //update end time
  md.fend_time = time(0);

  // convert start and end times into time format: Year-Month-DayTHours:Minutes:Seconds
  char endbuf[80], startbuf[80];
  struct tm tstruct;
  tstruct = *localtime(&md.fend_time);
  strftime(endbuf,sizeof(endbuf),"%Y-%m-%dT%H:%M:%S",&tstruct);
  tstruct = *localtime(&md.fstart_time);
  strftime(startbuf,sizeof(startbuf),"%Y-%m-%dT%H:%M:%S",&tstruct);
  
  // open a json file and write everything from the struct md complying to the 
  // samweb json format. This json file holds the below information temporarily. 
  // If you submitted a grid job invoking this service, the information from 
  // this file is appended to a final json file and this file will be removed

  std::ofstream jsonfile;
  jsonfile.open(fJSONFileName);
  jsonfile<<"{\n  \"application\": {\n    \"family\": "<<std::get<0>(md.fapplication)<<",\n    \"name\": ";
  jsonfile<<std::get<1>(md.fapplication)<<",\n    \"version\": "<<std::get<2>(md.fapplication)<<"\n  },\n  ";
  jsonfile<<"\"data_tier\": \""<<md.fdata_tier<<"\",\n  ";
  jsonfile<<"\"event_count\": "<<md.fnew_event_count<<",\n  ";
  //jsonfile<<"\"fcl.name\": \""<<md.ffcl_name<<"\",\n  ";
  //jsonfile<<"\"fcl.version\":  \""<<md.ffcl_version<<"\",\n  ";
  jsonfile<<"\"file_format\": \""<<md.ffile_format<<"\",\n  ";
  jsonfile<<"\"file_type\": "<<md.ffile_type<<",\n  ";
  jsonfile<<"\"first_event\": "<<md.ffirst_event<<",\n  ";
  jsonfile<<"\"group\": "<<md.fgroup<<",\n  ";
  jsonfile<<"\"last_event\": "<<md.flast_event<<",\n  ";
  //if (md.fdataTier != "generated"){
  unsigned int c=0;
  jsonfile<<"\"parents\": [\n";
  for(auto parent : md.fParents) {
    c++;
    size_t n = parent.find_last_of('/');
    size_t f1 = (n == std::string::npos ? 0 : n+1);
    jsonfile<<"    {\n     \"file_name\": \""<<parent.substr(f1)<<"\"\n    }";
    if (md.fParents.size()==1 || c==md.fParents.size()) jsonfile<<"\n";
    else jsonfile<<",\n"; 
  }      
  jsonfile<<"  ],\n  "; 
  //}   
  c=0;
  jsonfile<<"\"runs\": [\n";
  for(auto &t : md.fruns){
    c++;
    jsonfile<<"    [\n     "<<std::get<0>(t)<<",\n     "<<std::get<1>(t)<<",\n     "<<std::get<2>(t)<<"\n    ]";
    if (md.fruns.size()==1 || c==md.fruns.size()) jsonfile<<"\n";
    else jsonfile<<",\n"; 
  }
  jsonfile<<"  ],\n";          

  if (md.fMCGenerators!="") jsonfile << "\"lbne_MC.generators\": \"" << md.fMCGenerators << "\",\n";
  if (md.fMCOscillationP!="") jsonfile << "\"lbne_MC.oscillationP\": \"" << md.fMCOscillationP << "\",\n";
  if (md.fMCTriggerListVersion!="") jsonfile << "\"lbne_MC.trigger-list-version\": \"" << md.fMCTriggerListVersion << "\",\n";
  if (md.fMCBeamEnergy!="") jsonfile << "\"lbne_MC.beam_energy\": \"" << md.fMCBeamEnergy << "\",\n";
  if (md.fMCBeamFluxID!="") jsonfile << "\"lbne_MC.beam_flux_ID\": \"" << md.fMCBeamFluxID << "\",\n";
  if (md.fMCName!="") jsonfile << "\"lbne_MC.name\": \"" << md.fMCName << "\",\n";
  if (md.fMCDetectorType!="") jsonfile << "\"lbne_MC.detector_type\": \"" << md.fMCDetectorType << "\",\n";
  if (md.fMCNeutrinoFlavors!="") jsonfile << "\"lbne_MC.neutrino_flavors\": \"" << md.fMCNeutrinoFlavors << "\",\n";
  if (md.fMCMassHierarchy!="") jsonfile << "\"lbne_MC.mass_hierarchy\": \"" << md.fMCMassHierarchy << "\",\n";
  if (md.fMCMiscellaneous!="") jsonfile << "\"lbne_MC.miscellaneous\": \"" << md.fMCMiscellaneous << "\",\n";
  if (md.fMCGeometryVersion!="") jsonfile << "\"lbne_MC.geometry_version\": \"" << md.fMCGeometryVersion << "\",\n";
  if (md.fMCOverlay!="") jsonfile << "\"lbne_MC.overlay\": \"" << md.fMCOverlay << "\",\n";
  if (md.fDataRunMode!="") jsonfile << "\"lbne_data.run_mode\": \"" << md.fDataRunMode << "\",\n";
  if (md.fDataDetectorType!="") jsonfile << "\"lbne_data.detector_type\": \"" << md.fDataDetectorType << "\",\n";
  if (md.fDataName!="") jsonfile << "\"lbne_data.name\": \"" << md.fDataName << "\",\n";
  // fStageName appears not to be in our metadata spec

  // put these at the end because we know they'll be there and the last one needs to not have a comma

  jsonfile<<"\"start_time\": \""<<startbuf<<"\",\n";
  jsonfile<<"\"end_time\": \""<<endbuf<<"\"\n";
  
  //jsonfile<<"  \"ub_project.name\": \""<<md.fproject_name<<"\",\n  ";
  //jsonfile<<"\"ub_project.stage\": \""<<md.fproject_stage;
  //jsonfile<<"\",\n  \"ub_project.version\": \""<<md.fproject_version<<"\"\n";
  
  jsonfile<<"}\n";
  jsonfile.close();  
}

void util::TFileMetadataDUNE::postEvent ( art::Event const &  ev, art::ScheduleContext    )

private

Definition at line 191 of file TFileMetadataDUNE_service.cc.

{
 
  if(!fGenerateTFileMetadata) return;   
  
  art::RunNumber_t run = evt.run();
  art::SubRunNumber_t subrun = evt.subRun();
  art::EventNumber_t event = evt.event();
  art::SubRunID srid = evt.id().subRunID();
      
  // save run, subrun and runType information once every subrun    
  if (fSubRunNumbers.count(srid) == 0){
    fSubRunNumbers.insert(srid);
    md.fruns.push_back(make_tuple(run, subrun, frunType));   
  }
  
  // save the first event
  if (md.fevent_count == 0) md.ffirst_event = event;
  md.flast_event = event;
  // event counter
  ++md.fevent_count;

  art::Handle<art::TriggerResults> h;
  if (evt.getByLabel("TriggerResults", h) and h->accept()) {
    // Event passed at least one of the paths
    ++md.fnew_event_count;
  }
    
}

void util::TFileMetadataDUNE::postOpenFile ( std::string const &  fn )

private

Definition at line 180 of file TFileMetadataDUNE_service.cc.

{
  if (!fGenerateTFileMetadata) return;
  
  // save parent input files here
  md.fParents.insert(fn);
  
}

void util::TFileMetadataDUNE::reconfigure

(

fhicl::ParameterSet const &

p

)

Definition at line 97 of file TFileMetadataDUNE_service.cc.

{    
  // Get parameters.
  fGenerateTFileMetadata = pset.get<bool>("GenerateTFileMetadata", false);
  fJSONFileName = pset.get<std::string>("JSONFileName");
    
  if (!fGenerateTFileMetadata) return;

  md.fdata_tier            = pset.get<std::string>("dataTier","root-tuple");       
  md.ffile_format          = pset.get<std::string>("fileFormat","root");              
}

Member Data Documentation

bool util::TFileMetadataDUNE::fGenerateTFileMetadata

private

Definition at line 92 of file TFileMetadataDUNE.h.

std::string util::TFileMetadataDUNE::fJSONFileName

private

Definition at line 94 of file TFileMetadataDUNE.h.

std::string util::TFileMetadataDUNE::frunType

private

Definition at line 93 of file TFileMetadataDUNE.h.

std::set<art::SubRunID> util::TFileMetadataDUNE::fSubRunNumbers

Definition at line 74 of file TFileMetadataDUNE.h.

metadata util::TFileMetadataDUNE::md

Definition at line 73 of file TFileMetadataDUNE.h.

std::map<std::string,std::string> util::TFileMetadataDUNE::mdmap

Definition at line 75 of file TFileMetadataDUNE.h.

---

The documentation for this class was generated from the following files:

• dunecore/dunecore/Utilities/TFileMetadataDUNE.h
• dunecore/dunecore/Utilities/TFileMetadataDUNE_service.cc