Public Member Functions | Public Attributes | List of all members
extractor_prod.expMetaData Class Reference
Inheritance diagram for extractor_prod.expMetaData:
extractor_prod.MetaData

Public Member Functions

def __init__ (self, expname, inputfile)
 
def translateKey (self, key)
 
def md_gen (self, mdart, md0={})
 
def getmetadata (self, md0={})
 
- Public Member Functions inherited from extractor_prod.MetaData
def __init__ (self, inputfile)
 
def extract_metadata_to_pipe (self)
 
def get_job (self, proc)
 

Public Attributes

 expname
 
 translateKeyf
 
 md
 
- Public Attributes inherited from extractor_prod.MetaData
 inputfile
 

Additional Inherited Members

- Static Public Member Functions inherited from extractor_prod.MetaData
def wait_for_subprocess (jobinfo, q)
 
def mdart_gen (jobtuple)
 
def md_handle_application (md)
 
- Static Public Attributes inherited from extractor_prod.MetaData
string mdtext = ''
 mdtext = ''.join(line.replace(", ,", ",") for line in jobtuple[0].split('
') if line[-3:-1] != ' ,') More...
 
 mdtop = json.JSONDecoder().decode(mdtext)
 
 file_name = list(mdtop.keys())[0]
 

Detailed Description

Class to hold/interpret experiment-specific metadata

Definition at line 124 of file extractor_prod.py.

Constructor & Destructor Documentation

def extractor_prod.expMetaData.__init__ (   self,
  expname,
  inputfile 
)

Definition at line 126 of file extractor_prod.py.

126  def __init__(self, expname, inputfile):
127  MetaData.__init__(self, inputfile)
128  self.expname = expname
129  #self.exp_md_keyfile = expname + '_metadata_key'
130 # try:
131 # #translateMetaData = __import__("experiment_utilities", "MetaDataKey")
132 # from experiment_utilities import MetaDataKey
133 # except ImportError:
134 # print("You have not defined an experiment-specific metadata and key-translating module in experiment_utilities. Exiting")
135 # raise
136 #
137  metaDataModule = MetaDataKey()
138  self.metadataList, self.translateKeyf = metaDataModule.metadataList(), metaDataModule.translateKey
139 
extractor_prod.expMetaData.expname
expname
Definition: extractor_prod.py:128
extractor_prod.MetaDataKey
Definition: extractor_prod.py:103
extractor_prod.expMetaData.__init__
def __init__(self, expname, inputfile)
Definition: extractor_prod.py:126
extractor_prod.expMetaData.translateKeyf
translateKeyf
Definition: extractor_prod.py:138

Member Function Documentation

def extractor_prod.expMetaData.getmetadata (   self,
  md0 = {} 
) 
Get metadata from input file and return as python dictionary.
Calls other methods in class and returns metadata dictionary

Definition at line 340 of file extractor_prod.py.

340  def getmetadata(self, md0={}):
341  """ Get metadata from input file and return as python dictionary.
342  Calls other methods in class and returns metadata dictionary"""
343  proc = self.extract_metadata_to_pipe()
344  jobt = self.get_job(proc)
345  mdart = self.mdart_gen(jobt)
346  return self.md_gen(mdart, md0)
347 
extractor_prod.MetaData.get_job
def get_job(self, proc)
Definition: extractor_prod.py:55
extractor_prod.MetaData.extract_metadata_to_pipe
def extract_metadata_to_pipe(self)
Definition: extractor_prod.py:41
extractor_prod.expMetaData.md_gen
def md_gen(self, mdart, md0={})
Definition: extractor_prod.py:144
extractor_prod.expMetaData.getmetadata
def getmetadata(self, md0={})
Definition: extractor_prod.py:340
extractor_prod.MetaData.mdart_gen
def mdart_gen(jobtuple)
Definition: extractor_prod.py:82
def extractor_prod.expMetaData.md_gen (   self,
  mdart,
  md0 = {} 
) 
Loop through art metdata, generate metadata dictionary

Definition at line 144 of file extractor_prod.py.

144  def md_gen(self, mdart, md0={}):
145  """Loop through art metdata, generate metadata dictionary"""
146  # define an empty python dictionary which will hold sam metadata.
147  # Some fields can be copied directly from art metadata to sam metadata.
148  # Other fields require conversion.
149  md = {}
150 
151  # Loop over art metadata.
152  for mdkey in list(mdart.keys()):
153  mdval = mdart[mdkey]
154 
155  # Skip some art-specific fields.
156  if mdkey == 'file_format_version':
157  pass
158  elif mdkey == 'file_format_era':
159  pass
160 
161  # Ignore primary run_type field (if any).
162  # Instead, get run_type from runs field.
163 
164  elif mdkey == 'run_type':
165  pass
166  elif mdkey == 'application.version':
167  pass
168  elif mdkey == 'application.family':
169  pass
170  elif mdkey == 'application.name':
171  pass
172 
173  # do not Ignore data_stream any longer.
174 
175  elif mdkey == 'data_stream':
176  if 'dunemeta.data_stream' not in list(mdart.keys()): # only use this data_stream value if dunemeta.data_stream is not present
177  md['data_stream'] = mdval
178 
179  # Ignore process_name as of 2018-09-22 because it is not in SAM yet
180  elif mdkey == 'process_name':
181 # md['process_name'] = mdval
182  pass
183  # Application family/name/version.
184 
185  elif mdkey == 'applicationFamily':
186  if not md.has_key('application'):
187  md['application'] = {}
188  md['application']['family'] = mdval
189  elif mdkey == 'StageName' or mdkey == 'applicationName':
190  if not md.has_key('application'):
191  md['application'] = {}
192  md['application']['name'] = mdval
193  elif mdkey == 'applicationVersion':
194  if not md.has_key('application'):
195  md['application'] = {}
196  md['application']['version'] = mdval
197 
198  # Parents.
199 
200  elif mdkey == 'parents':
201  mdparents = []
202  if not args.strip_parents:
203  for parent in mdval:
204  parent_dict = {'file_name': parent}
205  mdparents.append(parent_dict)
206  md['parents'] = mdparents
207 
208  # Other fields where the key or value requires minor conversion.
209 
210  elif mdkey == 'art.first_event':
211  md[mdkey] = mdval[2]
212  elif mdkey == 'art.last_event':
213  md[mdkey] = mdval[2]
214  elif mdkey == 'first_event':
215  md[mdkey] = mdval
216  elif mdkey == 'last_event':
217  md[mdkey] = mdval
218  elif mdkey == 'lbneMCGenerators':
219  md['lbne_MC.generators'] = mdval
220  elif mdkey == 'lbneMCOscillationP':
221  md['lbne_MC.oscillationP'] = mdval
222  elif mdkey == 'lbneMCTriggerListVersion':
223  md['lbne_MC.trigger-list-version'] = mdval
224  elif mdkey == 'lbneMCBeamEnergy':
225  md['lbne_MC.beam_energy'] = mdval
226  elif mdkey == 'lbneMCBeamFluxID':
227  md['lbne_MC.beam_flux_ID'] = mdval
228  elif mdkey == 'lbneMCName':
229  md['lbne_MC.name'] = mdval
230  elif mdkey == 'lbneMCDetectorType':
231  md['lbne_MC.detector_type'] = mdval
232  elif mdkey == 'lbneMCNeutrinoFlavors':
233  md['lbne_MC.neutrino_flavors'] = mdval
234  elif mdkey == 'lbneMCMassHierarchy':
235  md['lbne_MC.mass_hierarchy'] = mdval
236  elif mdkey == 'lbneMCMiscellaneous':
237  md['lbne_MC.miscellaneous'] = mdval
238  elif mdkey == 'lbneMCGeometryVersion':
239  md['lbne_MC.geometry_version'] = mdval
240  elif mdkey == 'lbneMCOverlay':
241  md['lbne_MC.overlay'] = mdval
242  elif mdkey == 'lbneDataRunMode':
243  md['lbne_data.run_mode'] = mdval
244  elif mdkey == 'lbneDataDetectorType':
245  md['lbne_data.detector_type'] = mdval
246  elif mdkey == 'lbneDataName':
247  md['lbne_data.name'] = mdval
248  elif mdkey == 'detector.hv_status':
249  md[mdkey] = mdval
250  elif mdkey == 'detector.hv_value':
251  md[mdkey] = mdval
252  elif mdkey == 'detector.tpc_status':
253  md[mdkey] = mdval
254  elif mdkey == 'detector.tpc_apa_status':
255  md[mdkey] = mdval
256  elif mdkey == 'detector.tpc_apas':
257  md[mdkey] = mdval
258  elif mdkey == 'detector.tpc_apa_1':
259  md[mdkey] = mdval
260  elif mdkey == 'detector.tpc_apa_2':
261  md[mdkey] = mdval
262  elif mdkey == 'detector.tpc_apa_3':
263  md[mdkey] = mdval
264  elif mdkey == 'detector.tpc_apa_4':
265  md[mdkey] = mdval
266  elif mdkey == 'detector.tpc_apa_5':
267  md[mdkey] = mdval
268  elif mdkey == 'detector.tpc_apa_6':
269  md[mdkey] = mdval
270  elif mdkey == 'detector.pd_status':
271  md[mdkey] = mdval
272  elif mdkey == 'detector.crt_status':
273  md[mdkey] = mdval
274  elif mdkey == 'daq.readout':
275  md[mdkey] = mdval
276  elif mdkey == 'daq.felix_status':
277  md[mdkey] = mdval
278  elif mdkey == 'beam.polarity':
279  md[mdkey] = mdval
280  elif mdkey == 'beam.momentum':
281  md[mdkey] = mdval
282  elif mdkey == 'dunemeta.data_stream':
283  md['data_stream'] = mdval
284  elif mdkey == '??.data_type':
285  md[mdkey] = mdval
286  elif mdkey == 'data_quality.level':
287  md[mdkey] = mdval
288  elif mdkey == 'data_quality.is_junk':
289  md[mdkey] = mdval
290  elif mdkey == 'data_quality.do_not_process':
291  md[mdkey] = mdval
292  elif mdkey == 'data_quality.online_good_run_list':
293  md[mdkey] = mdval
294  elif mdkey == 'dunemeta.dune_data.accouple':
295  md['DUNE_data.accouple'] = int(mdval)
296  elif mdkey == 'dunemeta.dune_data.calibpulsemode':
297  md['DUNE_data.calibpulsemode'] = int(mdval)
298  elif mdkey == 'dunemeta.dune_data.daqconfigname':
299  md['DUNE_data.DAQConfigName'] = mdval
300  elif mdkey == 'dunemeta.dune_data.detector_config':
301  md['DUNE_data.detector_config'] = mdval
302  elif mdkey == 'dunemeta.dune_data.febaselinehigh':
303  md['DUNE_data.febaselinehigh'] = int(mdval)
304  elif mdkey == 'dunemeta.dune_data.fegain':
305  md['DUNE_data.fegain'] = int(mdval)
306  elif mdkey == 'dunemeta.dune_data.feleak10x':
307  md['DUNE_data.feleak10x'] = int(mdval)
308  elif mdkey == 'dunemeta.dune_data.feleakhigh':
309  md['DUNE_data.feleakhigh'] = int(mdval)
310  elif mdkey == 'dunemeta.dune_data.feshapingtime':
311  md['DUNE_data.feshapingtime'] = int(mdval)
312  elif mdkey == 'dunemeta.dune_data.inconsistent_hw_config':
313  md['DUNE_data.inconsistent_hw_config'] = int(mdval)
314  elif mdkey == 'dunemeta.dune_data.is_fake_data':
315  md['DUNE_data.is_fake_data'] = int(mdval)
316  elif mdkey == 'dunemeta.dune_data.readout_window':
317  md['DUNE_data.readout_window'] = float(mdval)
318  # For all other keys, copy art metadata directly to sam metadata.
319  # This works for run-tuple (run, subrun, runtype) and time stamps.
320 
321  else:
322  md[mdkey] = mdart[mdkey]
323 
324  # Get the other meta data field parameters
325 
326  md['file_name'] = self.inputfile.split("/")[-1]
327  if 'file_size' in md0:
328  md['file_size'] = md0['file_size']
329  else:
330  md['file_size'] = os.path.getsize(self.inputfile)
331  if 'crc' in md0 and not args.no_crc:
332  md['crc'] = md0['crc']
333  elif not args.no_crc:
334  md['crc'] = root_metadata.fileEnstoreChecksum(self.inputfile)
335 
336  # In case we ever want to check out what md is for any instance of MetaData by calling instance.md
337  self.md = md
338  return self.md
339 
reco_momentum_tuples.float
float
Definition: reco_momentum_tuples.py:12
extractor_prod.MetaData.inputfile
inputfile
Definition: extractor_prod.py:39
keras_to_tensorflow.int
int
Definition: keras_to_tensorflow.py:69
extractor_prod.expMetaData.md_gen
def md_gen(self, mdart, md0={})
Definition: extractor_prod.py:144
extractor_prod.expMetaData.md
md
Definition: extractor_prod.py:337
def extractor_prod.expMetaData.translateKey (   self,
  key 
) 
Returns the output of the imported translateKey function (as translateKeyf) called on key

Definition at line 140 of file extractor_prod.py.

140  def translateKey(self, key):
141  """Returns the output of the imported translateKey function (as translateKeyf) called on key"""
142  return self.translateKeyf(key)
143 
extractor_prod.expMetaData.translateKey
def translateKey(self, key)
Definition: extractor_prod.py:140
extractor_prod.expMetaData.translateKeyf
translateKeyf
Definition: extractor_prod.py:138

Member Data Documentation

extractor_prod.expMetaData.expname

Definition at line 128 of file extractor_prod.py.

extractor_prod.expMetaData.md

Definition at line 337 of file extractor_prod.py.

extractor_prod.expMetaData.translateKeyf

Definition at line 138 of file extractor_prod.py.

The documentation for this class was generated from the following file:
Generated by   doxygen 1.8.11