PDSPTPCRawDecoder_module.cc
Go to the documentation of this file.
1 ////////////////////////////////////////////////////////////////////////
2 // Class: PDSPTPCRawDecoder
3 // Plugin Type: producer (art v2_10_03)
4 // File: PDSPTPCRawDecoder_module.cc
5 //
6 // Generated at Fri Mar 2 15:36:20 2018 by Thomas Junk using cetskelgen
7 // from cetlib version v3_02_00.
8 // Original code from Jingbo Wang in separate RCE and FELIX raw decoders
9 // *********************************************************************
10 // July 2018, Maggie Greenwood, Added histograms for error checking.
11 // March 2019 restructure to read in lists of module label/instances of inputs, including
12 // an optional GetManyByType if we don't know in advance what labels we're going to see
13 ////////////////////////////////////////////////////////////////////////
14 
23 #include "fhiclcpp/ParameterSet.h"
26 #include "art_root_io/TFileService.h"
27 
28 #include <memory>
29 #include <cmath>
30 
31 // ROOT includes
32 #include "TH1.h"
33 #include "TStyle.h"
34 #include "TMath.h"
35 
36 // artdaq and dune-raw-data includes
39 #include "artdaq-core/Data/Fragment.hh"
40 #include "artdaq-core/Data/ContainerFragment.hh"
43 #include "dam/HeaderFragmentUnpack.hh"
44 #include "dam/DataFragmentUnpack.hh"
45 #include "dam/TpcFragmentUnpack.hh"
46 #include "dam/TpcStreamUnpack.hh"
47 #include "dam/access/WibFrame.hh"
48 #include "dam/access/Headers.hh"
49 #include "dam/access/TpcStream.hh"
50 #include "dam/access/TpcRanges.hh"
51 #include "dam/access/TpcToc.hh"
52 #include "dam/access/TpcPacket.hh"
53 #include "dam/RceFragmentUnpack.hh"
54 
55 // larsoft includes
58 #include "lardataobj/RawData/raw.h"
59 
60 // DUNE includes
62 
63 class PDSPTPCRawDecoder;
64 
66 
67 public:
68  explicit PDSPTPCRawDecoder(fhicl::ParameterSet const & p);
69  PDSPTPCRawDecoder(PDSPTPCRawDecoder const &) = delete;
73  void produce(art::Event & e) override;
74 
75 private:
76  typedef std::vector<raw::RawDigit> RawDigits;
77  typedef std::vector<raw::RDTimeStamp> RDTimeStamps;
81  typedef std::vector<raw::RDStatus> RDStatuses;
82 
83  // configuration parameters
84 
85  std::vector<int> _apas_to_decode;
86 
87  long int _min_offline_channel; // min offline channel to decode. <0: no limit
88  long int _max_offline_channel; // max offline channel to decode. <0: no limit. max<min: no limit
89 
92  std::vector<std::string> _rce_input_labels; // input labels also include instances. Example: "daq:TPC" or "daq::ContainerTPC"
93  std::vector<std::string> _felix_input_labels; // input labels also include instances. Example: "daq:FELIX" or "daq::ContainerFELIX"
100  unsigned int _full_channel_count;
103  unsigned int _full_tick_count;
115 
116  // what to do with unexpected crate numbers
117 
119 
120  // flags for attempting to fix FEMB 110's misaligned data
121 
123  unsigned int _rce_fix110_nticks;
124 
133 
136 
137  //declare histogram data memebers
139  unsigned int duplicate_channels;
140  unsigned int error_counter;
141  unsigned int incorrect_ticks;
142  unsigned int rcechans;
143  unsigned int felixchans;
145  //TH1I * fIncorrectTickNumbersZoomed;
146  TH1I * fParticipRCE;
150  TH1I * fFragSizeRCE;
152 
153  // flags and state needed for the data integrity enforcement mechanisms
154 
155  unsigned int _tick_count_this_event; // for use in comparing tick counts for all channels
158  const unsigned int _duplicate_channel_checklist_size=15360;
162 
163  // internal methods
164 
165  bool _processRCE(art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm);
166  bool _rceProcContNCFrags(art::Handle<artdaq::Fragments> frags, size_t &n_rce_frags, bool is_container,
167  art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm);
168  bool _process_RCE_AUX(const artdaq::Fragment& frag, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm, size_t ntickscheck);
169  void _process_RCE_nticksvf(const artdaq::Fragment& frag, std::vector<size_t> &nticksvec);
170 
171  bool _processFELIX(art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm);
172  bool _felixProcContNCFrags(art::Handle<artdaq::Fragments> frags, size_t &n_felix_frags, bool is_container, art::Event &evt, RawDigits& raw_digits,
173  RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm);
174  bool _process_FELIX_AUX(const artdaq::Fragment& frag, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm);
175 
176  void computeMedianSigma(raw::RawDigit::ADCvector_t &v_adc, float &median, float &sigma);
177 
178  std::vector<int16_t> _buffer;
179 };
180 
181 
183 {
184  std::vector<int> emptyivec;
185  _apas_to_decode = p.get<std::vector<int> >("APAsToDecode",emptyivec);
186  _default_crate_if_unexpected = p.get<unsigned int>("DefaultCrateIfUnexpected",3);
187 
188  _rce_input_labels.resize(0);
189  _rce_useInputLabels = p.get_if_present<std::vector<std::string> >("RCERawDataLabels",_rce_input_labels);
190  if (!_rce_useInputLabels)
191  {
192  _rce_input_labels.resize(0);
193  }
194  if (_rce_input_labels.size() == 0) // handle case where array is in the fcl document, just is empty
195  {
196  _rce_useInputLabels = false;
197  }
198  _rce_enforce_fragment_type_match = p.get<bool>("RCEEnforceFragmentTypeMatch",false);
199  _rce_fragment_type = p.get<int>("RCEFragmentType",2);
200  _drop_events_with_small_rce_frags = p.get<bool>("RCEDropEventsWithSmallFrags",false);
201  _drop_small_rce_frags = p.get<bool>("RCEDropSmallFrags",true);
202  _rce_frag_small_size = p.get<unsigned int>("RCESmallFragSize",10000);
203  _rce_drop_frags_with_badsf = p.get<bool>("RCEDropFragsWithBadSF",true);
204  _rce_drop_frags_with_badc = p.get<bool>("RCEDropFragsWithBadC",true);
205  _rce_hex_dump = p.get<bool>("RCEHexDump",false);
206  _rce_save_frags_to_files = p.get<bool>("RCESaveFragsToFiles",false);
207  _rce_check_buffer_size = p.get<bool>("RCECheckBufferSize",true);
208  _rce_buffer_size_checklimit = p.get<unsigned int>("RCEBufferSizeCheckLimit",10000000);
209 
210  // parameters to steer the FEMB 110 band-aid
211 
212  _rce_fix110 = p.get<bool>("RCEFIX110",true);
213  _rce_fix110_nticks = p.get<unsigned int>("RCEFIX110NTICKS",18);
214 
215  _felix_input_labels.resize(0);
216  _felix_useInputLabels = p.get_if_present<std::vector<std::string> >("FELIXRawDataLabels",_felix_input_labels);
218  {
219  _felix_input_labels.resize(0);
220  }
221  if (_felix_input_labels.size() == 0) // handle case where array is in the fcl document, just is empty
222  {
223  _felix_useInputLabels = false;
224  }
225 
226  _felix_enforce_fragment_type_match = p.get<bool>("FELIXEnforceFragmentTypeMatch",false);
227  _felix_fragment_type = p.get<int>("FELIXFragmentType",8);
228  _felix_drop_frags_with_badsf = p.get<bool>("FELIXDropFragsWithBadSF",true);
229  _felix_drop_frags_with_badc = p.get<bool>("FELIXDropFragsWithBadC",true);
230  _felix_hex_dump = p.get<bool>("FELIXHexDump",false);
231  _drop_events_with_small_felix_frags = p.get<bool>("FELIXDropEventsWithSmallFrags",false);
232  _drop_small_felix_frags = p.get<bool>("FELIXDropSmallFrags",true);
233  _felix_frag_small_size = p.get<unsigned int>("FELIXSmallFragSize",10000);
234  _felix_check_buffer_size = p.get<bool>("FELIXCheckBufferSize",true);
235  _felix_buffer_size_checklimit = p.get<unsigned int>("FELIXBufferSizeCheckLimit",10000000);
236 
237  _output_label = p.get<std::string>("OutputDataLabel");
238 
239  _enforce_full_channel_count = p.get<bool>("EnforceFullChannelCount", false);
240  _full_channel_count = p.get<unsigned int>("FullChannelCount", 15360);
241  _enforce_same_tick_count = p.get<bool>("EnforceSameTickCount",false);
242  _enforce_full_tick_count = p.get<bool>("EnforceFullTickCount",false);
243  _full_tick_count = p.get<unsigned int>("FullTickCount",6000);
244  _enforce_error_free = p.get<bool>("EnforceErrorFree",false);
245  _enforce_no_duplicate_channels = p.get<bool>("EnforceNoDuplicateChannels", true);
246 
247  _compress_Huffman = p.get<bool>("CompressHuffman",false);
248  _print_coldata_convert_count = p.get<bool>("PrintColdataConvertCount",false);
249 
250  _min_offline_channel = p.get<long int>("MinOfflineChannel",-1);
251  _max_offline_channel = p.get<long int>("MaxOfflineChannel",-1);
252 
253  produces<RawDigits>( _output_label ); //the strings in <> are the typedefs defined above
254  produces<RDTimeStamps>( _output_label );
255  produces<RDTsAssocs>( _output_label );
256  produces<RDStatuses>( _output_label );
257 
258  //Initialize Histograms if the tag is present
259  //art::ServiceHandle<art::TFileService> fs;
260  _make_histograms = p.get<bool>("MakeHistograms",false);
261 
262  if (_make_histograms)
263  {
265 
266  //Number of channels with wrong number of tics plotted to have an adjusted log2 scale on x axis
267  fIncorrectTickNumbers = tFileService->make<TH1D>("fIncorrectTickNumbers","Channels with Unexpected Number of Ticks", 45, -0.5, 14.5);
268  fIncorrectTickNumbers->GetXaxis()->SetTitle("Channels with an Unexpected Number of Ticks");
269  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(2,"1");
270  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(8,"4");
271  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(14,"16");
272  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(23,"128");
273  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(32,"1024");
274  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(38,"4096");
275  fIncorrectTickNumbers->GetXaxis()->SetBinLabel(44, "16384");
276 
277  //same as fIncorrectTickNumbers but with a zoomed domain
278  //fIncorrectTickNumbersZoomed = tFileService->make<TH1I>("fIncorrectTickNumbersZoomed","Channels with Unexpected Number of Ticks", , 0.5, 100.5);
279  //fIncorrectTickNumbersZoomed->GetXaxis()->SetTitle("Channels with an Unexpected Number of Tics (log2)");
280 
281  //number of participating RCE channels per event
282  fParticipRCE = tFileService->make<TH1I>("fParticipRCE","Participating RCE channels", 130, 0.5, 15000.5); //expected value 128000
283  fParticipRCE->GetXaxis()->SetTitle("RCE channels");
284 
285  //number of participating FELIX channels per event
286  fParticipFELIX = tFileService->make<TH1I>("fParticipFELIX","Participating FELIX channels", 100, 0.5, 3000.5); //expected value 2560
287  fParticipFELIX->GetXaxis()->SetTitle("FELIX channels");
288 
289  //number of duplicated channels per event
290  fDuplicatesNumber = tFileService->make<TH1I>("fDuplicatesNumber", "Number of Duplucated Channels", 200, 0.5, 200.5);
291  fDuplicatesNumber->GetXaxis()->SetTitle("Number of Duplicates");
292  //gStyle->SetOptStat("nemro");
293 
294  //number of channels with error returns
295  fErrorsNumber = tFileService->make<TH1D>("fErrorsNumber", "Channels with Errors", 45, -0.5, 14.5);
296  fErrorsNumber->GetXaxis()->SetTitle("Number of channels with errors");
297  fErrorsNumber->GetXaxis()->SetBinLabel(2,"1");
298  fErrorsNumber->GetXaxis()->SetBinLabel(8,"4");
299  fErrorsNumber->GetXaxis()->SetBinLabel(14,"16");
300  fErrorsNumber->GetXaxis()->SetBinLabel(23,"128");
301  fErrorsNumber->GetXaxis()->SetBinLabel(32,"1024");
302  fErrorsNumber->GetXaxis()->SetBinLabel(38,"4096");
303  fErrorsNumber->GetXaxis()->SetBinLabel(44, "16384");
304 
305  //total fragment sizes
306  fFragSizeRCE = tFileService->make<TH1I>("fFragSizeRCE", "RCE Fragment Size", 100, 0.5, 288000000.5);
307  fFragSizeRCE->GetXaxis()->SetTitle("Size of RCE Fragments (bytes)");
308 
309  fFragSizeFELIX = tFileService->make<TH1I>("fFragSizeFELIX", "FELIX Fragment Size", 100, 0.5, 57600000.5);
310  fFragSizeFELIX->GetXaxis()->SetTitle("Size of FELIX Fragments (bytes)");
311  }
312 }
313 
315 {
316  RawDigits raw_digits;
317  RDTimeStamps rd_timestamps;
318  RDTsAssocs rd_ts_assocs;
319 
320  RDPmkr rdpm(e,_output_label);
321  TSPmkr tspm(e,_output_label);
322 
323  error_counter = 0; //reset the errors to zero for each run
324  incorrect_ticks = 0;
325  duplicate_channels = 0;
326  rcechans = 0;
327  felixchans = 0;
328 
330  _discard_data = false; // true if we're going to drop the whole event's worth of data
331  _DiscardedCorruptData = false; // can be set to true if we drop some of the event's data
332  _KeptCorruptData = false; // true if we identify a corruption candidate but are skipping the test to drop it
333 
334  for (size_t i=0; i<_duplicate_channel_checklist_size; ++i) _duplicate_channel_checklist[i] = false;
335 
336  _processRCE(e,raw_digits,rd_timestamps,rd_ts_assocs,rdpm,tspm);
337  _processFELIX(e,raw_digits,rd_timestamps,rd_ts_assocs,rdpm,tspm);
338 
339  //Make the histograms for error checking. (other histograms are filled within the _process and _AUX functions)
340  if(_make_histograms)
341  {
342  fErrorsNumber->Fill(log2(error_counter));
345  fParticipFELIX->Fill(felixchans);
346  fParticipRCE->Fill(rcechans);
347  //fIncorrectTickNumbersZoomed->Fill(incorrect_ticks);
348  }
349 
350  if (_enforce_full_channel_count && raw_digits.size() != _full_channel_count)
351  {
352  MF_LOG_WARNING("PDSPTPCRawDecoder:") << "Wrong Total number of Channels " << raw_digits.size()
353  << " which is not " << _full_channel_count << ". Discarding Data";
354  _DiscardedCorruptData = true;
355  _discard_data = true;
356  }
357 
358  if (_discard_data)
359  {
360  RawDigits empty_raw_digits;
361  RDTimeStamps empty_rd_timestamps;
362  RDTsAssocs empty_rd_ts_assocs;
363  RDStatuses statuses;
364  statuses.emplace_back(true,false,1);
365  e.put(std::make_unique<decltype(empty_raw_digits)>(std::move(empty_raw_digits)),_output_label);
366  e.put(std::make_unique<decltype(empty_rd_timestamps)>(std::move(empty_rd_timestamps)),_output_label);
367  e.put(std::make_unique<decltype(empty_rd_ts_assocs)>(std::move(empty_rd_ts_assocs)),_output_label);
368  e.put(std::make_unique<decltype(statuses)>(std::move(statuses)),_output_label);
369  }
370  else
371  {
372  RDStatuses statuses;
373  unsigned int statword=0;
374  if (_DiscardedCorruptData) statword |= 1;
375  if (_KeptCorruptData) statword |= 2;
376  statuses.emplace_back(_DiscardedCorruptData,_KeptCorruptData,statword);
377  e.put(std::make_unique<decltype(raw_digits)>(std::move(raw_digits)),_output_label);
378  e.put(std::make_unique<decltype(rd_timestamps)>(std::move(rd_timestamps)),_output_label);
379  e.put(std::make_unique<decltype(rd_ts_assocs)>(std::move(rd_ts_assocs)),_output_label);
380  e.put(std::make_unique<decltype(statuses)>(std::move(statuses)),_output_label);
381  }
382 }
383 
384 bool PDSPTPCRawDecoder::_processRCE(art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
385 {
386  size_t n_rce_frags = 0;
387  bool have_data=false;
388  bool have_data_nc=false;
389 
391  {
392  for (size_t ilabel = 0; ilabel < _rce_input_labels.size(); ++ ilabel)
393  {
394  if (_rce_input_labels.at(ilabel).find("Container") != std::string::npos)
395  {
396  auto cont_frags = evt.getHandle<artdaq::Fragments>(_rce_input_labels.at(ilabel));
397  if (cont_frags)
398  {
399  have_data = true;
400  if (! _rceProcContNCFrags(cont_frags, n_rce_frags, true, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
401  {
402  return false;
403  }
404  }
405  }
406  else
407  {
408  auto frags = evt.getHandle<artdaq::Fragments>(_rce_input_labels.at(ilabel));
409  if (frags)
410  {
411  have_data_nc = true;
412  if (! _rceProcContNCFrags(frags, n_rce_frags, false, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
413  {
414  return false;
415  }
416  }
417  }
418  }
419  }
420  else // get all the fragments in the event and look for the ones that say TPC in them
421  {
422  //std::vector<art::Handle<artdaq::Fragments> > fraghv; // fragment handle vector
423  //evt.getManyByType(fraghv);
424  auto fraghv = evt.getMany<artdaq::Fragments>();
425 
426  for (size_t ihandle=0; ihandle<fraghv.size(); ++ihandle)
427  {
428  if (fraghv.at(ihandle).provenance()->inputTag().instance().find("TPC") != std::string::npos)
429  {
430  if (fraghv.at(ihandle).isValid())
431  {
432  if (fraghv.at(ihandle).provenance()->inputTag().instance().find("Container") != std::string::npos)
433  {
434  have_data = true;
435  if (! _rceProcContNCFrags(fraghv.at(ihandle), n_rce_frags, true, evt, raw_digits, timestamps, tsassocs, rdpm, tspm) )
436  {
437  return false;
438  }
439  }
440  else
441  {
442  have_data_nc = true;
443  if (! _rceProcContNCFrags(fraghv.at(ihandle), n_rce_frags, false, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
444  {
445  return false;
446  }
447  }
448  }
449  }
450  }
451  }
452 
453  //MF_LOG_INFO("_processRCE")
454  //<< " Processed " << n_rce_frags
455  //<< " RCE Fragments, making "
456  //<< raw_digits.size()
457  //<< " RawDigits.";
458 
459  // returns true if we want to add to the number of fragments processed. Separate flag used
460  // for data error conditions (_discard_data).
461 
462  return have_data || have_data_nc;
463 }
464 
465 bool PDSPTPCRawDecoder::_rceProcContNCFrags(art::Handle<artdaq::Fragments> frags, size_t &n_rce_frags, bool is_container,
466  art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
467 {
468  //size of RCE fragments into histogram
469  if(_make_histograms)
470  {
471  size_t rcebytes = 0;
472  for (auto const& frag : *frags)
473  {
474  rcebytes = rcebytes + (frag.sizeBytes());
475  }
476  fFragSizeRCE->Fill(rcebytes);
477  }
478 
479 
480  // figure out what the median number of ticks is
481 
482  std::vector<size_t> nticksvec;
483  for (auto const& frag : *frags)
484  {
485  //std::cout << "RCE fragment size bytes: " << frag.sizeBytes() << std::endl;
486 
487  // skip small fragments even here
488 
490  {
491  if (is_container)
492  {
493  artdaq::ContainerFragment cont_frag(frag);
494  for (size_t ii = 0; ii < cont_frag.block_count(); ++ii)
495  {
496  _process_RCE_nticksvf(*cont_frag[ii], nticksvec);
497  }
498  }
499  else
500  {
501  _process_RCE_nticksvf(frag, nticksvec);
502  }
503  }
504  }
505  if (nticksvec.size() == 0)
506  {
507  MF_LOG_WARNING("_process_RCE:") << " No valid nticks to check. Discarding Event.";
508  _discard_data = true;
509  _DiscardedCorruptData = true;
510  evt.removeCachedProduct(frags);
511  return false;
512  }
513  size_t nticksmedian = TMath::Median(nticksvec.size(),nticksvec.data()) + 0.01; // returns a double -- want to make sure it gets truncated to the right integer
514 
515 
516  // actually process the
517  for (auto const& frag : *frags)
518  {
519  //std::cout << "RCE fragment size bytes: " << frag.sizeBytes() << std::endl;
520 
521  bool process_flag = true;
522  if (frag.sizeBytes() < _rce_frag_small_size)
523  {
525  {
526  MF_LOG_WARNING("_process_RCE:") << " Small RCE fragment size: " << frag.sizeBytes() << " Discarding Event on request.";
527  _discard_data = true;
528  _DiscardedCorruptData = true;
529  evt.removeCachedProduct(frags);
530  return false;
531  }
532  if ( _drop_small_rce_frags )
533  {
534  MF_LOG_WARNING("_process_RCE:") << " Small RCE fragment size: " << frag.sizeBytes() << " Discarding just this fragment on request.";
535  _DiscardedCorruptData = true;
536  process_flag = false;
537  }
538  _KeptCorruptData = true;
539  }
540  if (process_flag)
541  {
542  if (is_container)
543  {
544  artdaq::ContainerFragment cont_frag(frag);
545  for (size_t ii = 0; ii < cont_frag.block_count(); ++ii)
546  {
547  if (_process_RCE_AUX(*cont_frag[ii], raw_digits, timestamps, tsassocs, rdpm, tspm, nticksmedian)) ++n_rce_frags;
548  }
549  }
550  else
551  {
552  if (_process_RCE_AUX(frag, raw_digits, timestamps,tsassocs, rdpm, tspm, nticksmedian)) ++n_rce_frags;
553  }
554  }
555  }
556  evt.removeCachedProduct(frags);
557  return true;
558 }
559 
561  const artdaq::Fragment& frag,
562  std::vector<size_t> &nticksvec
563  )
564 {
565 
566  if (_rce_hex_dump)
567  {
568  std::ios oldState(nullptr);
569  oldState.copyfmt(std::cout);
570 
571  std::cout << "RCE Fragment: all numbers in hex " << std::hex
572  << " SequenceID = " << frag.sequenceID()
573  << " fragmentID = " << frag.fragmentID()
574  << " fragmentType = " << (unsigned)frag.type()
575  << " Timestamp = " << frag.timestamp() << std::endl;
576  std::cout << "Offset Data";
577  artdaq::Fragment fragloc(frag);
578  unsigned char *dbegin = reinterpret_cast<unsigned char *>(fragloc.dataAddress());
579  size_t dsize = fragloc.dataSizeBytes();
580  size_t offcounter=0;
581  for (size_t bcounter=0; bcounter<dsize;++bcounter)
582  {
583  if ( (offcounter % 8) == 0 )
584  {
585  std::cout << std::endl << std::hex << std::setfill('0') << std::setw(8) << offcounter << " ";
586  }
587  std::cout << std::hex << std::setfill('0') << std::setw(2) << (int) *dbegin << " ";
588  dbegin++;
589  offcounter++;
590  }
591  std::cout << std::endl;
592  std::cout.copyfmt(oldState);
593  }
594 
595  dune::RceFragment rce(frag);
596 
598  {
599  TString outfilename="rce";
600  outfilename += frag.sequenceID();
601  outfilename += "_";
602  outfilename += frag.fragmentID();
603  outfilename+=".fragment";
604  rce.save(outfilename.Data());
605  std::cout << "Saved an RCE fragment with " << rce.size() << " streams: " << outfilename << std::endl;
606  }
607 
608  artdaq::Fragment cfragloc(frag);
609  size_t cdsize = cfragloc.dataSizeBytes();
610  const uint64_t* cdptr = (uint64_t const*) (cfragloc.dataBeginBytes() + 12); // see dune-raw-data/Overlays/RceFragment.cc
611  HeaderFragmentUnpack const cdheader(cdptr);
612  //bool isOkay = RceFragmentUnpack::isOkay(cdptr,cdsize+sizeof(cdheader));
613  if (cdsize>16) cdsize -= 16;
614  bool isOkay = RceFragmentUnpack::isOkay(cdptr,cdsize);
615  if (!isOkay) return;
616 
617  // // skip if damaged but not FEMB 302
618  // if (cdheader.isData())
619  // {
620  // DataFragmentUnpack df(cdptr);
621  // if (df.isTpcDamaged())
622  // {
623  // bool found302 = false;
624  // for (int i = 0; i < rce.size(); ++i)
625  // {
626  // auto const * rce_stream = rce.get_stream(i);
627  // auto const identifier = rce_stream->getIdentifier();
628  // uint32_t crateNumber = identifier.getCrate();
629  // uint32_t slotNumber = identifier.getSlot();
630  // uint32_t fiberNumber = identifier.getFiber();
631  // if (crateNumber == 3 && slotNumber == 3 && fiberNumber == 2) found302 = true;
632  // }
633  // if (!found302) return;
634  // }
635  // }
636 
637  for (int i = 0; i < rce.size(); ++i)
638  {
639  auto const * rce_stream = rce.get_stream(i);
640  //size_t n_ch = rce_stream->getNChannels();
641  size_t n_ticks = rce_stream->getNTicks();
642  nticksvec.push_back(n_ticks);
643  }
644 
645 }
646 
648  const artdaq::Fragment& frag,
649  RawDigits& raw_digits,
650  RDTimeStamps &timestamps,
651  RDTsAssocs &tsassocs,
652  RDPmkr &rdpm, TSPmkr &tspm,
653  size_t ntickscheck
654  )
655 {
656 
657  if (_rce_enforce_fragment_type_match && (frag.type() != _rce_fragment_type))
658  {
659  MF_LOG_WARNING("_process_RCE_AUX:") << " RCE fragment type " << (int) frag.type() << " doesn't match expected value: " << _rce_fragment_type << " Discarding RCE fragment";
660  _DiscardedCorruptData = true;
661  return false;
662  }
663  //MF_LOG_INFO("_Process_RCE_AUX")
664  //<< " SequenceID = " << frag.sequenceID()
665  //<< " fragmentID = " << frag.fragmentID()
666  //<< " fragmentType = " << (unsigned)frag.type()
667  //<< " Timestamp = " << frag.timestamp();
669 
670  dune::RceFragment rce(frag);
671  artdaq::Fragment cfragloc(frag);
672  size_t cdsize = cfragloc.dataSizeBytes();
673  const uint64_t* cdptr = (uint64_t const*) (cfragloc.dataBeginBytes() + 12); // see dune-raw-data/Overlays/RceFragment.cc
674  HeaderFragmentUnpack const cdheader(cdptr);
675  //bool isOkay = RceFragmentUnpack::isOkay(cdptr,cdsize+sizeof(cdheader));
676  if (cdsize>16) cdsize -= 16;
677  bool isOkay = RceFragmentUnpack::isOkay(cdptr,cdsize);
678  if (!isOkay)
679  {
680  MF_LOG_WARNING("_process_RCE_AUX:") << "RCE Fragment isOkay failed: " << cdsize << " Discarding this fragment";
681  error_counter++;
682  _DiscardedCorruptData = true;
683  return false;
684  }
685 
686  //DataFragmentUnpack df(cdptr);
687  //std::cout << "isTPpcNormal: " << df.isTpcNormal() << " isTpcDamaged: " << df.isTpcDamaged() << " isTpcEmpty: " << df.isTpcEmpty() << std::endl;
688 
689 
690  uint32_t ch_counter = 0;
691  for (int i = 0; i < rce.size(); ++i)
692  {
693  auto const * rce_stream = rce.get_stream(i);
694  size_t n_ch = rce_stream->getNChannels();
695  size_t n_ticks = rce_stream->getNTicks();
696  if (n_ticks == 0) continue; // on David Adams's request
697  auto const identifier = rce_stream->getIdentifier();
698  uint32_t crateNumber = identifier.getCrate();
699  uint32_t slotNumber = identifier.getSlot();
700  uint32_t fiberNumber = identifier.getFiber();
701 
702  //std::cout << "Processing an RCE Stream: " << crateNumber << " " << slotNumber << " " << fiberNumber << " " << n_ticks << " " << n_ch << std::endl;
703 
704  size_t adsiz = _apas_to_decode.size();
705  bool apafound = true; // default if no vector to test against
706  if (adsiz)
707  {
708  apafound = false;
709  for (unsigned int j=0; j<adsiz; ++j)
710  {
711  if ( ((int) crateNumber == _apas_to_decode[j]) ||
712  (_apas_to_decode[j] < 0) ||
713  ( (crateNumber == 0 || crateNumber > 6) && _apas_to_decode[j] == 7) )
714  {
715  apafound = true;
716  break;
717  }
718  }
719  }
720  if (!apafound)
721  {
722  return false;
723  }
724 
725  // check for bad crate numbers -- default empty list of APAs to check, default check is on, and if crate number isn't
726  // one of the DAQ ones. If we asked for a crate and it's there, even if it's unusual, let it pass.
727 
728 
729  if ( (crateNumber == 0 || crateNumber > 6) && _rce_drop_frags_with_badc && adsiz == 0)
730  {
731  MF_LOG_WARNING("_process_RCE:") << "Bad crate number, discarding fragment on request: "
732  << (int) crateNumber;
733  return false;
734  }
735 
736  if (slotNumber > 4 || fiberNumber == 0 || fiberNumber > 4)
737  {
739  {
740  MF_LOG_WARNING("_process_RCE:") << "Bad slot, fiber number, discarding fragment on request: "
741  << " " << slotNumber << " " << fiberNumber;
742  _DiscardedCorruptData = true;
743  return false;
744  }
745  _KeptCorruptData = true;
746  }
747 
749  {
750  // from JJ's PdReaderTest.cc
751  using namespace pdd;
752  using namespace pdd::access;
753  bool printed=false;
754  TpcStream const &stream = rce_stream->getStream ();
755  TpcToc toc (stream.getToc ());
756  TpcPacket pktRec (stream.getPacket ());
757  TpcPacketBody pktBdy (pktRec.getRecord ());
758  int npkts = toc.getNPacketDscs ();
759  for (int ipkt = 0; ipkt < npkts; ++ipkt)
760  {
761  TpcTocPacketDsc pktDsc (toc.getPacketDsc (ipkt));
762  unsigned int o64 = pktDsc.getOffset64 ();
763  unsigned int pktType = pktDsc.getType ();
764  unsigned nWibFrames = pktDsc.getNWibFrames ();
765  WibFrame const *wf = pktBdy.getWibFrames (pktType, o64);
766  for (unsigned iwf = 0; iwf < nWibFrames; ++iwf)
767  {
768  auto const &colddata = wf->getColdData ();
769  auto cvt0 = colddata[0].getConvertCount ();
770  //auto cvt1 = colddata[1].getConvertCount ();
771  std::cout << "RCE coldata convert count: " << cvt0 << std::endl;
772  printed = true;
773  ++wf; // in case we were looping over WIB frames, but let's stop at the first
774  break;
775  }
776  if (printed) break;
777  }
778  }
779 
780 
781  if(_make_histograms)
782  {
783  //log the participating RCE channels
784  rcechans=rcechans+n_ch;
785  }
786 
787  // check the number of ticks and allow FEMB302 to have 10% fewer
788  size_t ntc10 = ( 0.9 * (float) ntickscheck );
789  if (!(
790  n_ticks == ntickscheck ||
791  (
792  (crateNumber == 3 && slotNumber == 3 && fiberNumber == 2) &&
793  n_ticks < ntickscheck && n_ticks > ntc10)
794  )
795  )
796  {
797  MF_LOG_WARNING("_process_RCE_AUX:") << "Nticks differs from median or FEMB302 nticks not expected: " << n_ticks << " "
798  << ntickscheck << " Discarding this fragment";
799  _DiscardedCorruptData = true;
800  return false;
801  }
802 
803  if (n_ticks != _full_tick_count)
804  {
806  {
807  MF_LOG_WARNING("_process_RCE_AUX:") << "Nticks not the required value: " << n_ticks << " "
808  << _full_tick_count << " Discarding Data";
809  error_counter++;
810  incorrect_ticks++;
811  _discard_data = true;
812  _DiscardedCorruptData = true;
813  return false;
814  }
815  _KeptCorruptData = true;
816  }
817 
819  {
821  _tick_count_this_event = n_ticks;
822  }
823  else
824  {
825  if (n_ticks != _tick_count_this_event)
826  {
828  {
829  MF_LOG_WARNING("_process_RCE_AUX:") << "Nticks different for two channel streams: " << n_ticks
830  << " vs " << _tick_count_this_event << " Discarding Data";
831  error_counter++;
832  _discard_data = true;
833  _DiscardedCorruptData = true;
834  return false;
835  }
836  }
837  _KeptCorruptData = true;
838  }
839 
840 
841  //MF_LOG_INFO("_Process_RCE_AUX")
842  //<< "RceFragment timestamp: " << rce_stream->getTimeStamp()
843  //<< ", NChannels: " << n_ch
844  //<< ", NTicks: " << n_ticks;
845 
846  // TODO -- speed this up!! Remove one buffer copy
847 
848  size_t buffer_size = n_ch * n_ticks;
849 
850  if (buffer_size > _rce_buffer_size_checklimit)
851  {
853  {
854  MF_LOG_WARNING("_process_RCE_AUX:") << "n_ch*nticks too large: " << n_ch << " * " << n_ticks << " = " <<
855  buffer_size << " larger than: " << _rce_buffer_size_checklimit << ". Discarding this fragment";
856  _DiscardedCorruptData = true;
857  return false;
858  }
859  else
860  {
861  _KeptCorruptData = true;
862  }
863  }
864 
865  if (_buffer.capacity() < buffer_size)
866  {
867  // MF_LOG_INFO("_process_RCE_AUX")
868  //<< "Increase buffer size from " << _buffer.capacity()
869  //<< " to " << buffer_size;
870 
871  _buffer.reserve(buffer_size);
872  }
873 
874  int16_t* adcs = _buffer.data();
875  bool sgmcdretcode = rce_stream->getMultiChannelData(adcs);
876  if (!sgmcdretcode)
877  {
879  {
880  MF_LOG_WARNING("_process_RCE_AUX:") << "getMutliChannelData returns error flag: "
881  << " c:s:f:ich: " << crateNumber << " " << slotNumber << " " << fiberNumber << " Discarding Data";
882  error_counter++;
883  _DiscardedCorruptData = true;
884  return false;
885  }
886  _KeptCorruptData = true;
887  }
888 
889  //std::cout << "RCE raw decoder trj: " << crateNumber << " " << slotNumber << " " << fiberNumber << std::endl;
890 
891  // David Adams's request for channels to start at zero for coldbox test data
892  unsigned int crateloc = crateNumber;
893  if (crateNumber == 0 || crateNumber > 6) crateloc = _default_crate_if_unexpected;
894 
896  for (size_t i_ch = 0; i_ch < n_ch; i_ch++)
897  {
898  unsigned int offlineChannel = channelMap->GetOfflineNumberFromDetectorElements(crateloc, slotNumber, fiberNumber, i_ch, dune::PdspChannelMapService::kRCE);
899 
900  // skip this channel if we are asked to.
901 
903  (offlineChannel < (size_t) _min_offline_channel || offlineChannel > (size_t) _max_offline_channel) ) continue;
904 
905  v_adc.clear();
906 
907  if (_rce_fix110 && crateNumber == 1 && slotNumber == 0 && fiberNumber == 1 && channelMap->ChipFromOfflineChannel(offlineChannel) == 4 && n_ticks > _rce_fix110_nticks)
908  {
909  for (size_t i_tick = 0; i_tick < n_ticks-_rce_fix110_nticks; i_tick++)
910  {
911  v_adc.push_back(adcs[i_tick+_rce_fix110_nticks]);
912  }
913  for (size_t i_tick=0; i_tick<_rce_fix110_nticks; ++i_tick)
914  {
915  v_adc.push_back(v_adc.back());
916  }
917 
918  }
919  else
920  {
921  for (size_t i_tick = 0; i_tick < n_ticks; i_tick++)
922  {
923  v_adc.push_back(adcs[i_tick]);
924  }
925  }
926  adcs += n_ticks;
927 
928  ch_counter++;
929 
930  if (offlineChannel < _duplicate_channel_checklist_size)
931  {
932  if (_duplicate_channel_checklist[offlineChannel])
933  {
934  if(_make_histograms)
935  {
937  }
938 
940  {
941  MF_LOG_WARNING("_process_RCE_AUX:") << "Duplicate Channel: " << offlineChannel
942  << " c:s:f:ich: " << crateNumber << " " << slotNumber << " " << fiberNumber << " " << i_ch << " Discarding Data";
943  error_counter++;
944  _discard_data = true;
945  _DiscardedCorruptData = true;
946  return false;
947  }
948  _KeptCorruptData = true;
949  }
950  _duplicate_channel_checklist[offlineChannel] = true;
951  }
952 
953  float median=0;
954  float sigma=0;
955  computeMedianSigma(v_adc,median,sigma);
956 
957  /// FEMB 302 IS crate 3, slot 3, fiber 2
958 
959  auto uncompressed_nticks = v_adc.size(); // can be different from n_ticks due to padding of FEMB 302
960 
962  if (_compress_Huffman)
963  {
964  cflag = raw::kHuffman;
965  raw::Compress(v_adc,cflag);
966  }
967  // here n_ticks is the uncompressed size as required by the constructor
968  raw::RawDigit raw_digit(offlineChannel, uncompressed_nticks, v_adc, cflag);
969  raw_digit.SetPedestal(median,sigma);
970  raw_digits.push_back(raw_digit);
971 
972  raw::RDTimeStamp rdtimestamp(rce_stream->getTimeStamp(),offlineChannel);
973  timestamps.push_back(rdtimestamp);
974 
975  //associate the raw digit and the timestamp data products
976  auto const rawdigitptr = rdpm(raw_digits.size()-1);
977  auto const rdtimestampptr = tspm(timestamps.size()-1);
978  tsassocs.addSingle(rawdigitptr,rdtimestampptr);
979  }
980  }
981 
982  return true;
983 }
984 
985 
986 
987 bool PDSPTPCRawDecoder::_processFELIX(art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
988 {
989  size_t n_felix_frags = 0;
990  bool have_data=false;
991  bool have_data_nc=false;
992 
994  {
995  for (size_t ilabel = 0; ilabel < _felix_input_labels.size(); ++ ilabel)
996  {
997  if (_felix_input_labels.at(ilabel).find("Container") != std::string::npos)
998  {
999  auto cont_frags = evt.getHandle<artdaq::Fragments>(_felix_input_labels.at(ilabel));
1000  if (cont_frags)
1001  {
1002  have_data = true;
1003  if (! _felixProcContNCFrags(cont_frags, n_felix_frags, true, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
1004  {
1005  return false;
1006  }
1007  }
1008  }
1009  else
1010  {
1011  auto frags = evt.getHandle<artdaq::Fragments>(_felix_input_labels.at(ilabel));
1012  if (frags)
1013  {
1014  have_data_nc = true;
1015  if (! _felixProcContNCFrags(frags, n_felix_frags, false, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
1016  {
1017  return false;
1018  }
1019  }
1020  }
1021  }
1022  }
1023  else // get all the fragments in the event and look for the ones that say FELIX in them
1024  {
1025  //std::vector<art::Handle<artdaq::Fragments> > fraghv; // fragment handle vector
1026  //evt.getManyByType(fraghv);
1027  auto fraghv = evt.getMany<artdaq::Fragments>();
1028 
1029  for (size_t ihandle=0; ihandle<fraghv.size(); ++ihandle)
1030  {
1031  if (fraghv.at(ihandle).provenance()->inputTag().instance().find("FELIX") != std::string::npos)
1032  {
1033  if (fraghv.at(ihandle).isValid())
1034  {
1035  if (fraghv.at(ihandle).provenance()->inputTag().instance().find("Container") != std::string::npos)
1036  {
1037  have_data = true;
1038  if (! _felixProcContNCFrags(fraghv.at(ihandle), n_felix_frags,true, evt, raw_digits, timestamps, tsassocs, rdpm, tspm) )
1039  {
1040  return false;
1041  }
1042  }
1043  else
1044  {
1045  have_data_nc = true;
1046  if (! _felixProcContNCFrags(fraghv.at(ihandle), n_felix_frags, false, evt, raw_digits, timestamps, tsassocs, rdpm, tspm))
1047  {
1048  return false;
1049  }
1050  }
1051  }
1052 
1053  // we had swept in all the TPC fragments, possibly for a second time, so remove them
1054  // be even more aggressive and remove all cached fragments -- anyone who needs them
1055  // should read them in, and getManyByType had swept them all into memory. Awaiting
1056  // a getManyLabelsByType if we go this route
1057 
1058  else // if (fraghv.at(ihandle).provenance()->inputTag().instance().find("TPC") != std::string::npos)
1059  {
1060  evt.removeCachedProduct(fraghv.at(ihandle));
1061  }
1062  }
1063  }
1064  }
1065 
1066  //MF_LOG_INFO("_processFELIX")
1067  //<< " Processed " << n_felix_frags
1068  //<< " FELIX Fragments, making "
1069  //<< raw_digits.size()
1070  //<< " RawDigits.";
1071 
1072  // returns true if we want to add to the number of fragments processed. Separate flag used
1073  // for data error conditions (_discard_data).
1074 
1075  return have_data || have_data_nc;
1076 }
1077 
1078 bool PDSPTPCRawDecoder::_felixProcContNCFrags(art::Handle<artdaq::Fragments> frags, size_t &n_felix_frags, bool is_container,
1079  art::Event &evt, RawDigits& raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
1080 {
1081  //size of FELIX fragments into histogram
1082  if(_make_histograms)
1083  {
1084  size_t felixbytes = 0;
1085  for (auto const& frag : *frags)
1086  {
1087  felixbytes = felixbytes + (frag.sizeBytes());
1088  }
1089  fFragSizeFELIX->Fill(felixbytes);
1090  }
1091 
1092  for (auto const& frag : *frags)
1093  {
1094  //std::cout << "FELIX fragment size bytes: " << frag.sizeBytes() << std::endl;
1095 
1096  bool process_flag = true;
1097  if (frag.sizeBytes() < _felix_frag_small_size)
1098  {
1100  {
1101  MF_LOG_WARNING("_process_FELIX:") << " Small FELIX fragment size: " << frag.sizeBytes() << " Discarding Event on request.";
1102  _discard_data = true;
1103  _DiscardedCorruptData = true;
1104  evt.removeCachedProduct(frags);
1105  return false;
1106  }
1108  {
1109  MF_LOG_WARNING("_process_FELIX:") << " Small FELIX fragment size: " << frag.sizeBytes() << " Discarding just this fragment on request.";
1110  _DiscardedCorruptData = true;
1111  process_flag = false;
1112  }
1113  _KeptCorruptData = true;
1114  }
1115  if (process_flag)
1116  {
1117  if (is_container)
1118  {
1119  artdaq::ContainerFragment cont_frag(frag);
1120  for (size_t ii = 0; ii < cont_frag.block_count(); ++ii)
1121  {
1122  if (_process_FELIX_AUX(*cont_frag[ii], raw_digits, timestamps, tsassocs, rdpm, tspm)) ++n_felix_frags;
1123  }
1124  }
1125  else
1126  {
1127  if (_process_FELIX_AUX(frag, raw_digits, timestamps,tsassocs, rdpm, tspm)) ++n_felix_frags;
1128  }
1129  }
1130  }
1131  evt.removeCachedProduct(frags);
1132  return true;
1133 }
1134 
1135 
1136 bool PDSPTPCRawDecoder::_process_FELIX_AUX(const artdaq::Fragment& frag, RawDigits& raw_digits,
1137  RDTimeStamps &timestamps,
1138  RDTsAssocs &tsassocs,
1139  RDPmkr &rdpm, TSPmkr &tspm)
1140 {
1141 
1142  //std::cout
1143  //<< " SequenceID = " << frag.sequenceID()
1144  //<< " fragmentID = " << frag.fragmentID()
1145  //<< " fragmentType = " << (unsigned)frag.type()
1146  //<< " Timestamp = " << frag.timestamp() << std::endl;
1147 
1148  if (_felix_hex_dump)
1149  {
1150  std::ios oldState(nullptr);
1151  oldState.copyfmt(std::cout);
1152 
1153  std::cout << "FELIX Fragment: all numbers in hex " << std::hex
1154  << " SequenceID = " << frag.sequenceID()
1155  << " fragmentID = " << frag.fragmentID()
1156  << " fragmentType = " << (unsigned)frag.type()
1157  << " Timestamp = " << frag.timestamp() << std::endl;
1158  std::cout << "Offset Data";
1159  artdaq::Fragment fragloc(frag);
1160  unsigned char *dbegin = reinterpret_cast<unsigned char *>(fragloc.dataAddress());
1161  size_t dsize = fragloc.dataSizeBytes();
1162  size_t offcounter=0;
1163  for (size_t bcounter=0; bcounter<dsize;++bcounter)
1164  {
1165  if ( (offcounter % 8) == 0 )
1166  {
1167  std::cout << std::endl << std::hex << std::setfill('0') << std::setw(8) << offcounter << " ";
1168  }
1169  std::cout << std::hex << std::setfill('0') << std::setw(2) << (int) *dbegin << " ";
1170  dbegin++;
1171  offcounter++;
1172  }
1173  std::cout << std::endl;
1174  std::cout.copyfmt(oldState);
1175  }
1176 
1177  // check against _felix_fragment_type
1178  if ( _felix_enforce_fragment_type_match && (frag.type() != _felix_fragment_type) )
1179  {
1180  _DiscardedCorruptData = true;
1181  MF_LOG_WARNING("_process_FELIX_AUX:") << " FELIX fragment type " << (int) frag.type() << " doesn't match expected value: " << _felix_fragment_type << " Discarding FELIX fragment";
1182  return false;
1183  }
1184 
1186 
1187  //Load overlay class.
1188  dune::FelixFragment felix(frag);
1189 
1190  //Get detector element numbers from the fragment
1191 
1192  uint8_t crate = felix.crate_no(0);
1193  uint8_t slot = felix.slot_no(0);
1194  uint8_t fiber = felix.fiber_no(0); // decode this one later
1195 
1196  size_t adsiz = _apas_to_decode.size();
1197  bool apafound = true; // default if no vector to test against
1198  if (adsiz)
1199  {
1200  apafound = false;
1201  for (unsigned int j=0; j<adsiz; ++j)
1202  {
1203  if (
1204  ((int) crate == _apas_to_decode[j]) ||
1205  (_apas_to_decode[j] < 0) ||
1206  ( (crate == 0 || crate > 6) && _apas_to_decode[j] == 7) ) // on David Adams's request for coldbox test data
1207  {
1208  apafound = true;
1209  break;
1210  }
1211  }
1212  }
1213  if (!apafound)
1214  {
1215  return false;
1216  }
1217 
1218  // check for bad crate numbers -- default empty list of APAs to check, default check is on, and if crate number isn't
1219  // one of the DAQ ones. If we asked for a crate and it's there, even if it's unusual, let it pass.
1220 
1221  if ( (crate == 0 || crate > 6) && _felix_drop_frags_with_badc && adsiz == 0)
1222  {
1223  MF_LOG_WARNING("_process_FELIX:") << "Bad crate number, discarding fragment on request: "
1224  << (int) crate;
1225  return false;
1226  }
1227 
1228  if ( slot > 4)
1229  {
1230  if (_felix_drop_frags_with_badsf) // we'll check the fiber later
1231  {
1232  _DiscardedCorruptData = true;
1233  MF_LOG_WARNING("_process_FELIX_AUX:") << "Invalid slot: s=" << (int) slot << " discarding FELIX data.";
1234  return false;
1235  }
1236  _KeptCorruptData = true;
1237  }
1238 
1240  {
1241  uint16_t first_coldata_convert_count = felix.coldata_convert_count(0,0);
1242  std::cout << "FELIX Coldata convert count: " << (int) first_coldata_convert_count << std::endl;
1243  }
1244 
1245  //std::cout << "FELIX raw decoder trj: " << (int) crate << " " << (int) slot << " " << (int) fiber << std::endl;
1246 
1247  const unsigned n_frames = felix.total_frames(); // One frame contains 25 felix (20 ns-long) ticks. A "frame" is an offline tick
1248  //std::cout<<" Nframes = "<<n_frames<<std::endl;
1249  //_h_nframes->Fill(n_frames);
1250  if (n_frames == 0) return true;
1251  const unsigned n_channels = dune::FelixFrame::num_ch_per_frame;// should be 256
1252 
1253 
1254  if (n_frames*n_channels > _felix_buffer_size_checklimit)
1255  {
1257  {
1258  MF_LOG_WARNING("_process_FELIX_AUX:") << "n_channels*n_frames too large: " << n_channels << " * " << n_frames << " = " <<
1259  n_frames*n_channels << " larger than: " << _felix_buffer_size_checklimit << ". Discarding this fragment";
1260  _DiscardedCorruptData = true;
1261  return false;
1262  }
1263  else
1264  {
1265  _KeptCorruptData = true;
1266  }
1267  }
1268 
1269  if(_make_histograms)
1270  {
1271  felixchans=felixchans+n_channels;
1272  }
1273 
1274  for (unsigned int iframe=0; iframe<n_frames; ++iframe)
1275  {
1276  if ( felix.wib_errors(iframe) != 0)
1277  {
1278  if (_enforce_error_free )
1279  {
1280  _DiscardedCorruptData = true;
1281  MF_LOG_WARNING("_process_FELIX_AUX:") << "WIB Errors on frame: " << iframe << " : " << felix.wib_errors(iframe)
1282  << " Discarding Data";
1283  error_counter++;
1284  // drop just this fragment
1285  //_discard_data = true;
1286  return true;
1287  }
1288  _KeptCorruptData = true;
1289  }
1290  }
1291 
1292  // check optimization of this -- size not reserved
1293 
1295  //v_adc.reserve(n_frames*n_channels);
1296  // Fill the adc vector.
1297 
1298  for(unsigned ch = 0; ch < n_channels; ++ch) {
1299 
1300  // handle 256 channels on two fibers -- use the channel map that assumes 128 chans per fiber (=FEMB)
1301 
1302  unsigned int fiberloc = 0;
1303  if (fiber == 1)
1304  {
1305  fiberloc = 1;
1306  }
1307  else if (fiber == 2)
1308  {
1309  fiberloc = 3;
1310  }
1311  else
1312  {
1313  MF_LOG_WARNING("_process_FELIX_AUX:") << " Fiber number " << (int) fiber << " is expected to be 1 or 2 -- revisit logic";
1314  fiberloc = 1;
1315  error_counter++;
1317  {
1318  MF_LOG_WARNING("_process_FELIX_AUX:") << " Dropping FELIX Data";
1319  return false;
1320  }
1321  }
1322 
1323  unsigned int chloc = ch;
1324  if (chloc > 127)
1325  {
1326  chloc -= 128;
1327  fiberloc++;
1328  }
1329  unsigned int crateloc = crate;
1330 
1331  // David Adams's request for channels to start at zero for coldbox test data
1332  if (crateloc == 0 || crateloc > 6) crateloc = _default_crate_if_unexpected;
1333 
1334  unsigned int offlineChannel = channelMap->GetOfflineNumberFromDetectorElements(crateloc, slot, fiberloc, chloc, dune::PdspChannelMapService::kFELIX);
1335 
1336  // skip this channel if we are asked to.
1337 
1339  (offlineChannel < (size_t) _min_offline_channel || offlineChannel > (size_t) _max_offline_channel) ) continue;
1340 
1341  v_adc.clear();
1342  //std::cout<<"crate:slot:fiber = "<<crate<<", "<<slot<<", "<<fiber<<std::endl;
1343  std::vector<dune::adc_t> waveform( felix.get_ADCs_by_channel(ch) );
1344  for(unsigned int nframe=0;nframe<waveform.size();nframe++){
1345  // if(ch==0 && nframe<100) {
1346  // if(nframe==0) std::cout<<"Print the first 100 ADCs of Channel#1"<<std::endl;
1347  // std::cout<<waveform.at(nframe)<<" ";
1348  // if(nframe==99) std::cout<<std::endl;
1349  // }
1350  v_adc.push_back(waveform.at(nframe));
1351  }
1352 
1353  if ( v_adc.size() != _full_tick_count)
1354  {
1356  {
1357  MF_LOG_WARNING("_process_FELIX_AUX:") << "Nticks not the required value: " << v_adc.size() << " "
1358  << _full_tick_count << " Discarding Data";
1359  error_counter++;
1360  incorrect_ticks++;
1361  _discard_data = true;
1362  _DiscardedCorruptData = true;
1363  return true;
1364  }
1365  _KeptCorruptData = true;
1366  }
1367 
1369  {
1371  _tick_count_this_event = v_adc.size();
1372  }
1373  else
1374  {
1376  {
1377  if (v_adc.size() != _tick_count_this_event)
1378  {
1379  MF_LOG_WARNING("_process_FELIX_AUX:") << "Nticks different for two channel streams: " << v_adc.size()
1380  << " vs " << _tick_count_this_event << " Discarding Data";
1381  error_counter++;
1382  _discard_data = true;
1383  _DiscardedCorruptData = true;
1384  return true;
1385  }
1386  _KeptCorruptData = true;
1387  }
1388  }
1389 
1390  if (offlineChannel < _duplicate_channel_checklist_size)
1391  {
1392  if (_duplicate_channel_checklist[offlineChannel])
1393  {
1394  if(_make_histograms)
1395  {
1397  }
1399  {
1400  MF_LOG_WARNING("_process_FELIX_AUX:") << "Duplicate Channel: " << offlineChannel
1401  << " c:s:f:ich: " << (int) crate << " " << (int) slot << " " << (int) fiber << " " << (int) ch << " Discarding Data";
1402  error_counter++;
1403  _discard_data = true;
1404  _DiscardedCorruptData = true;
1405  return true;
1406  }
1407  _KeptCorruptData = true;
1408  }
1409  _duplicate_channel_checklist[offlineChannel] = true;
1410  }
1411 
1412  float median=0;
1413  float sigma=0;
1414  computeMedianSigma(v_adc,median,sigma);
1415 
1416  auto n_ticks = v_adc.size();
1418  if (_compress_Huffman)
1419  {
1420  cflag = raw::kHuffman;
1421  raw::Compress(v_adc,cflag);
1422  }
1423  // here n_ticks is the uncompressed size as required by the constructor
1424  raw::RawDigit raw_digit(offlineChannel, n_ticks, v_adc, cflag);
1425  raw_digit.SetPedestal(median,sigma);
1426  raw_digits.push_back(raw_digit);
1427 
1428  raw::RDTimeStamp rdtimestamp(felix.timestamp(),offlineChannel);
1429  timestamps.push_back(rdtimestamp);
1430 
1431  //associate the raw digit and the timestamp data products
1432  auto const rawdigitptr = rdpm(raw_digits.size()-1);
1433  auto const rdtimestampptr = tspm(timestamps.size()-1);
1434  tsassocs.addSingle(rawdigitptr,rdtimestampptr);
1435  }
1436  return true;
1437 }
1438 
1439 
1440 // compute median and sigma. Sigma is half the distance between the upper and lower bounds of the
1441 // 68% region where 34% is above the median and 34% is below ("centered" on the median).
1442 
1444 {
1445  size_t asiz = v_adc.size();
1446  int imed=0;
1447  if (asiz == 0)
1448  {
1449  median = 0;
1450  sigma = 0;
1451  }
1452  else
1453  {
1454  // this is actually faster than the code below by about one second per event.
1455  // the RMS includes tails from bad samples and signals and may not be the best RMS calc.
1456 
1457  imed = TMath::Median(asiz,v_adc.data()) + 0.01; // add an offset to make sure the floor gets the right integer
1458  median = imed;
1459  sigma = TMath::RMS(asiz,v_adc.data());
1460 
1461  // add in a correction suggested by David Adams, May 6, 2019
1462 
1463  size_t s1 = 0;
1464  size_t sm = 0;
1465  for (size_t i=0; i<asiz; ++i)
1466  {
1467  if (v_adc[i] < imed) s1++;
1468  if (v_adc[i] == imed) sm++;
1469  }
1470  if (sm > 0)
1471  {
1472  float mcorr = (-0.5 + (0.5*(float) asiz - (float) s1)/ ((float) sm) );
1473  //if (std::abs(mcorr)>1.0) std::cout << "mcorr: " << mcorr << std::endl;
1474  median += mcorr;
1475  }
1476  }
1477 
1478 
1479 
1480  // never do this, but keep the code around in case we want it later
1481 
1482  // if (asiz > 100000000)
1483  // {
1484  // size_t mednum = asiz/2;
1485  // size_t m1snum = mednum - ( (float) asiz )*0.34;
1486  // size_t p1snum = mednum + ( (float) asiz )*0.34;
1487 
1488  // std::map<size_t,size_t> adcmap;
1489  // for (auto const adc : v_adc)
1490  // {
1491  // auto mapiter = adcmap.find(adc);
1492  // if (mapiter != adcmap.end())
1493  // {
1494  // mapiter->second ++;
1495  // }
1496  // else
1497  // {
1498  // adcmap[adc] = 1;
1499  // }
1500  // }
1501 
1502  // // find quantiles, -1 sigma, median, plus 1 sigma
1503  // size_t sum = 0;
1504  // size_t m1s = 0;
1505  // size_t p1s = 0;
1506  // size_t m = 0;
1507  // for (auto const &mv : adcmap)
1508  // {
1509  // sum += mv.second;
1510  // if (m1s == 0 && sum >= m1snum)
1511  // {
1512  // m1s = mv.first;
1513  // }
1514  // if (m == 0 && sum >= mednum)
1515  // {
1516  // m = mv.first;
1517  // }
1518  // if (p1s == 0 && sum >= p1snum)
1519  // {
1520  // p1s = mv.first;
1521  // break;
1522  // }
1523  // }
1524  // median = (float) m;
1525  // sigma = ((float) (p1s - m1s))/2.0;
1526  // }
1527 }
1528 
static constexpr size_t num_ch_per_frame
Definition: FelixFormat.hh:298
Huffman Encoding.
Definition: RawTypes.h:10
size_t total_frames() const
enum raw::_compress Compress_t
bool _felixProcContNCFrags(art::Handle< artdaq::Fragments > frags, size_t &n_felix_frags, bool is_container, art::Event &evt, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
Collection of charge vs time digitized from a single readout channel.
Definition: RawDigit.h:69
TpcStreamUnpack const * get_stream(int i) const
Definition: RceFragment.cc:48
uint8_t slot_no(const unsigned &frame_ID=0) const
Handle< PROD > getHandle(SelectorBase const &) const
Definition: DataViewImpl.h:382
std::string string
Definition: nybbler.cc:12
uint8_t crate_no(const unsigned &frame_ID=0) const
std::vector< raw::RDStatus > RDStatuses
EDProducer(fhicl::ParameterSet const &pset)
Definition: EDProducer.h:20
bool _duplicate_channel_checklist[15360]
std::vector< short > ADCvector_t
Type representing a (compressed) vector of ADC counts.
Definition: RawDigit.h:73
art::PtrMaker< raw::RawDigit > RDPmkr
bool _processFELIX(art::Event &evt, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
QTextStream & hex(QTextStream &s)
unsigned int _default_crate_if_unexpected
std::vector< raw::RawDigit > RawDigits
int size() const
Definition: RceFragment.hh:25
no compression
Definition: RawTypes.h:9
std::vector< std::string > _rce_input_labels
art::Assns< raw::RawDigit, raw::RDTimeStamp > RDTsAssocs
unsigned int GetOfflineNumberFromDetectorElements(unsigned int crate, unsigned int slot, unsigned int fiber, unsigned int fembchannel, FelixOrRCE frswitch)
bool _rceProcContNCFrags(art::Handle< artdaq::Fragments > frags, size_t &n_rce_frags, bool is_container, art::Event &evt, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
bool _process_RCE_AUX(const artdaq::Fragment &frag, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm, size_t ntickscheck)
bool _processRCE(art::Event &evt, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
const double e
#define DEFINE_ART_MODULE(klass)
Definition: ModuleMacros.h:67
std::vector< int16_t > _buffer
struct dune::tde::crate crate
art::PtrMaker< raw::RDTimeStamp > TSPmkr
std::vector< Handle< PROD > > getMany(SelectorBase const &selector=MatchAllSelector{}) const
Definition: DataViewImpl.h:479
def move(depos, offset)
Definition: depos.py:107
p
Definition: test.py:223
ProductID put(std::unique_ptr< PROD > &&edp, std::string const &instance={})
Definition: DataViewImpl.h:686
void produce(art::Event &e) override
void computeMedianSigma(raw::RawDigit::ADCvector_t &v_adc, float &median, float &sigma)
uint16_t coldata_convert_count(const unsigned &frame_ID, const uint8_t &block_num) const
std::vector< int > _apas_to_decode
bool removeCachedProduct(Handle< PROD > &) const
Definition: DataViewImpl.h:927
Q_EXPORT QTSManip setw(int w)
Definition: qtextstream.h:331
void save(const std::string &filepath) const
Definition: RceFragment.cc:63
void _process_RCE_nticksvf(const artdaq::Fragment &frag, std::vector< size_t > &nticksvec)
uint8_t fiber_no(const unsigned &frame_ID=0) const
adc_v get_ADCs_by_channel(const uint8_t channel_ID) const
uint16_t wib_errors(const unsigned &frame_ID=0) const
PDSPTPCRawDecoder(fhicl::ParameterSet const &p)
void SetPedestal(float ped, float sigma=1.)
Set pedestal and its RMS (the latter is 0 by default)
Definition: RawDigit.cxx:68
void addSingle(Ptr< left_t > const &left, Ptr< right_t > const &right, data_t const &data)
Definition: Assns.h:546
void Compress(std::vector< short > &adc, raw::Compress_t compress)
Compresses a raw data buffer.
Definition: raw.cxx:19
def access(path, mode)
PDSPTPCRawDecoder & operator=(PDSPTPCRawDecoder const &)=delete
TCEvent evt
Definition: DataStructs.cxx:7
#define MF_LOG_WARNING(category)
bool _process_FELIX_AUX(const artdaq::Fragment &frag, RawDigits &raw_digits, RDTimeStamps &timestamps, RDTsAssocs &tsassocs, RDPmkr &rdpm, TSPmkr &tspm)
Q_EXPORT QTSManip setfill(int f)
Definition: qtextstream.h:337
const unsigned int _duplicate_channel_checklist_size
std::vector< Fragment > Fragments
Definition: HDF5Utils.h:57
unsigned int ChipFromOfflineChannel(unsigned int offlineChannel) const
Returns chip number.
double median(sqlite3 *db, std::string const &table_name, std::string const &column_name)
Definition: statistics.cc:26
QTextStream & endl(QTextStream &s)
std::vector< raw::RDTimeStamp > RDTimeStamps
std::vector< std::string > _felix_input_labels
uint64_t timestamp(const unsigned &frame_ID=0) const