MixFilterTest_module.cc
Go to the documentation of this file.
2 
15 #include "cetlib/map_vector.h"
16 #include "fhiclcpp/types/Atom.h"
20 
21 #include <algorithm>
22 #include <iostream>
23 #include <iterator>
24 #include <memory>
25 #include <unordered_set>
26 
27 namespace arttest {
28  class MixFilterTestDetail;
29 #if ART_TEST_EVENTS_TO_SKIP_CONST
30 #define ART_MFT MixFilterTestETSc
31 #define ART_TEST_EVENTS_TO_SKIP_CONST_TXT const
32 #elif defined ART_TEST_EVENTS_TO_SKIP_CONST
33 #define ART_MFT MixFilterTestETS
34 #define ART_TEST_EVENTS_TO_SKIP_CONST_TXT
35 #elif defined ART_TEST_OLD_STARTEVENT
36 #define ART_MT MixFilterTestOldStartEvent
37 #elif defined ART_TEST_NO_STARTEVENT
38 #define ART_MT MixFilterTestNoStartEvent
39 #else
40 // Normal case
41 #define ART_MFT MixFilterTest
42 #endif
44 }
45 
46 namespace {
47  class SecondaryFileNameProvider {
48  public:
49  SecondaryFileNameProvider(std::vector<std::string>&& fileNames)
50  : fileNames_{move(fileNames)}, fileNameIter_{fileNames_.cbegin()}
51  {}
52 
53  SecondaryFileNameProvider(SecondaryFileNameProvider const& other)
54  : fileNames_{other.fileNames_}
55  , fileNameIter_{fileNames_.cbegin() +
56  (other.fileNameIter_ - other.fileNames_.cbegin())}
57  {}
58 
59  SecondaryFileNameProvider&
60  operator=(SecondaryFileNameProvider const& other)
61  {
62  SecondaryFileNameProvider tmp{other};
63  std::swap(tmp, *this);
64  return *this;
65  }
66 
67  ~SecondaryFileNameProvider() noexcept = default;
68 
70  operator()()
71  {
72  if (fileNameIter_ == fileNames_.end()) {
73  return {};
74  } else {
75  return *(fileNameIter_++);
76  }
77  }
78 
79  private:
80  std::vector<std::string> fileNames_;
81  decltype(fileNames_.cbegin()) fileNameIter_;
82  };
83 } // namespace
84 
85 using namespace fhicl;
86 
88 public:
92 
93  struct Config {
94  Atom<size_t> numSecondaries{Name{"numSecondaries"}, 1};
95  Atom<bool> testRemapper{Name{"testRemapper"}, true};
96  Atom<bool> testZeroSecondaries{Name{"testZeroSecondaries"}, false};
97  Atom<bool> testPtrFailure{Name{"testPtrFailure"}, false};
98  Atom<bool> testEventOrdering{Name{"testEventOrdering"}, false};
99  Atom<bool> testNoLimEventDupes{Name{"testNoLimEventDupes"}, false};
100  Atom<bool> compactMissingProducts{Name{"compactMissingProducts"}, false};
101  Atom<size_t> expectedRespondFunctionCalls{
102  Name{"expectedRespondFunctionCalls"},
103  4ul};
104  OptionalSequence<std::string> fileNamesToProvide{
105  Name{"fileNamesToProvide"}};
106  Atom<std::string> mixProducerLabel{Name{"mixProducerLabel"}, "mixProducer"};
107  };
108 
110 
111  // Constructor is responsible for registering mix operations with
112  // MixHelper::declareMixOp() and bookkeeping products with
113  // MixHelper::produces().
115 
116  MixFilterTestDetail(MixFilterTestDetail const&) = delete;
118 
120 
121 #ifdef ART_TEST_OLD_STARTEVENT
122  // Old startEvent signature -- check it still works
123  void startEvent();
124 #elif !defined ART_TEST_NO_STARTEVENT
125  // Optional startEvent(Event const&): initialize state for each event,
126  void startEvent(art::Event const&);
127 #endif
128 
129  // Return the number of secondaries to read this time. Declare const
130  // if you don't plan to change your class' state.
131  size_t nSecondaries() const;
132 
133  // Optional eventsToSkip(): number of events to skip at the start of
134  // the file.
135 #ifdef ART_TEST_EVENTS_TO_SKIP_CONST
136  size_t
137  eventsToSkip() ART_TEST_EVENTS_TO_SKIP_CONST_TXT
138  {
139  return 7;
140  }
141 #endif
142 
143  // Optional processEventIDs(): after the generation of the event
144  // sequence, this function will be called if it exists to provide
145  // the sequence of EventIDs.
146  void processEventIDs(art::EventIDSequence const& seq);
147 
148  // Optional processEventAuxiliaries(): after the generation of the
149  // event sequence, this function will be called if it exists to
150  // provide the sequence of EventAuxiliaries.
152 
153  // Optional.finalizeEvent(): (eg) put bookkeeping products in
154  // event. Do *not* place mix products into the event: this will
155  // already have been done for you.
156  void finalizeEvent(art::Event& t);
157 
158  // Optional respondToXXXfunctions, called at the right time if they
159  // exist.
160  void respondToOpenInputFile(art::FileBlock const& fb);
161  void respondToCloseInputFile(art::FileBlock const& fb);
164 
165  // Optional {begin,end}{Sub,}Run functions, called at the right time
166  // if they exist.
167  void beginSubRun(art::SubRun const& sr);
168  void endSubRun(art::SubRun& sr);
169  void beginRun(art::Run const& r);
170  void endRun(art::Run& r);
171 
172  // Mixing functions. Note that they do not *have* to be member
173  // functions of this detail class: they may be member functions of a
174  // completely unrelated class; free functions or function objects
175  // provided they (or the function object's operator()) have the
176  // expected signature.
177  template <typename PROD, typename OPROD = PROD>
178  bool mixByAddition(std::vector<PROD const*> const&,
179  OPROD&,
180  art::PtrRemapper const&);
181 
183  std::vector<std::vector<double> const*> const& in,
184  std::vector<double>& out,
185  art::PtrRemapper const&);
186 
187  bool aggregate_map_vector(std::vector<mv_t const*> const& in,
188  mv_t& out,
189  art::PtrRemapper const&);
190 
191  bool mixPtrs(std::vector<std::vector<art::Ptr<double>> const*> const& in,
193  art::PtrRemapper const& remap);
194 
195 #ifndef ART_NO_MIX_PTRVECTOR
196  bool mixPtrVectors(std::vector<art::PtrVector<double> const*> const& in,
198  art::PtrRemapper const& remap);
199 #endif
200 
201  bool mixProductWithPtrs(
202  std::vector<arttest::ProductWithPtrs const*> const& in,
204  art::PtrRemapper const& remap);
205 
206  bool mixmap_vectorPtrs(
210  art::PtrRemapper const& remap);
211 
212  bool mixSRProduct(std::vector<double const*> const& in,
213  DoubleProduct& out,
214  art::PtrRemapper const&);
215 
216  bool mixRProduct(std::vector<double const*> const& in,
217  DoubleProduct& out,
218  art::PtrRemapper const&);
219 
220  template <typename COLL>
221  void verifyInSize(COLL const& in) const;
222 
223 private:
224  size_t const nSecondaries_;
225  bool const testRemapper_;
226  std::vector<size_t> doubleVectorOffsets_{};
227  std::vector<size_t> map_vectorOffsets_{};
228  std::unique_ptr<art::EventIDSequence> eIDs_{};
229  bool startEvent_called_{false};
235  size_t endRunCounter_{};
236  int currentEvent_{-1};
238  bool const testPtrFailure_;
239  bool const testEventOrdering_;
244 
246 
247  // For testing no_replace mode only:
248  std::vector<int> allEvents_{};
249  std::unordered_set<int> uniqueEvents_{};
250 
251  // For testing run and subrun mixing.
252  double subRunInfo_{};
253  double runInfo_{};
254 };
255 
256 template <typename COLL>
257 inline void
259 {
260  BOOST_REQUIRE_EQUAL(
261  in.size(),
263 }
264 
266  art::MixHelper& helper)
267  : nSecondaries_{p().numSecondaries()}
268  , testRemapper_{p().testRemapper()}
269  , testZeroSecondaries_{p().testZeroSecondaries()}
270  , testPtrFailure_{p().testPtrFailure()}
271  , testEventOrdering_{p().testEventOrdering()}
272  , testNoLimEventDupes_{p().testNoLimEventDupes()}
273  , compactMissingProducts_{p().compactMissingProducts()}
274  , expectedRespondFunctionCalls_{p().expectedRespondFunctionCalls()}
275  , readMode_{helper.readMode()}
276 {
277  std::vector<std::string> fnToProvide;
278  if (p().fileNamesToProvide(fnToProvide)) {
279  std::cerr << "Calling registerSecondaryFileNameProvider.\n";
280  std::copy(fnToProvide.cbegin(),
281  fnToProvide.cend(),
282  std::ostream_iterator<std::string>(std::cerr, ", "));
283  std::cerr << "\n";
284  helper.registerSecondaryFileNameProvider(
285  SecondaryFileNameProvider{move(fnToProvide)});
286  }
287 
288  // Test createEngine when it should have already been created
290  helper.createEngine(123456);
291  helper.createEngine(123456, "HepJamesRandom");
292  helper.createEngine(123456, "HepJamesRandom", "");
293  // Cannot assign empty label to different type
294  BOOST_REQUIRE_THROW(helper.createEngine(123456, "MTwistEngine"),
296  // Cannot specify type that is not supported.
297  BOOST_REQUIRE_THROW(helper.createEngine(123456, "HEPJamesRandom", "Edna"),
299  }
300 
301  auto const mixProducerLabel = p().mixProducerLabel();
302  helper.produces<std::string>(); // "Bookkeeping"
303  helper.produces<art::EventIDSequence>(); // "Bookkeeping"
304  helper.produces<double, art::InSubRun>(); // SubRun product test.
305  helper.produces<double, art::InRun>(); // Run product test.
306  helper.declareMixOp(art::InputTag{mixProducerLabel, "doubleLabel"},
307  &MixFilterTestDetail::mixByAddition<double>,
308  *this);
309  helper.declareMixOp(art::InputTag{mixProducerLabel, "IntProductLabel"},
310  &MixFilterTestDetail::mixByAddition<arttest::IntProduct>,
311  *this);
312  helper.declareMixOp(art::InputTag{mixProducerLabel, "stringLabel", "SWRITE"},
313  &MixFilterTestDetail::mixByAddition<std::string>,
314  *this);
315  helper.declareMixOp(art::InputTag{mixProducerLabel, "doubleCollectionLabel"},
317  *this);
318  helper.declareMixOp(art::InputTag{mixProducerLabel, "doubleVectorPtrLabel"},
320  *this);
321 #ifndef ART_NO_MIX_PTRVECTOR
322  helper.declareMixOp(art::InputTag{mixProducerLabel, "doublePtrVectorLabel"},
324  *this);
325 #endif
326  helper.declareMixOp(art::InputTag{mixProducerLabel, "ProductWithPtrsLabel"},
328  *this);
329  helper.declareMixOp(art::InputTag{mixProducerLabel, "mapVectorLabel"},
331  *this);
332  helper.declareMixOp(art::InputTag{mixProducerLabel, "intVectorPtrLabel"},
334  *this);
335  art::MixFunc<IntProduct> mixfunc(
336  [this](std::vector<IntProduct const*> const& in,
337  IntProduct,
338  art::PtrRemapper const&) -> bool {
339  auto const sz = in.size();
340  auto expected = nSecondaries_;
342  expected -= std::count_if(
343  eIDs_->begin(), eIDs_->end(), [](art::EventID const& eID) {
344  return (eID.event() % 100) == 0;
345  });
346  }
347  for (auto i = 0ul; i < sz; ++i) {
348  if (!compactMissingProducts_ && ((*eIDs_)[i].event() % 100) == 0) {
349  // Product missing
350  BOOST_REQUIRE(in[i] == nullptr);
351  } else {
352  BOOST_REQUIRE(in[i] != nullptr);
353  }
354  }
355  BOOST_REQUIRE_EQUAL(
356  sz, (currentEvent_ == 2 && testZeroSecondaries_) ? 0ul : expected);
357  return false;
358  });
359  helper.declareMixOp(
360  art::InputTag{mixProducerLabel, "SpottyProductLabel"}, mixfunc, false);
361  // SubRun mixing.
362  helper.declareMixOp<art::InSubRun>(
363  art::InputTag{mixProducerLabel, "DoubleSRLabel"},
364  "SRInfo",
366  *this);
367  // Run mixing.
368  helper.declareMixOp<art::InRun>(
369  art::InputTag{mixProducerLabel, "DoubleRLabel"},
370  "RInfo",
372  *this);
373 }
374 
376 {
378  testNoLimEventDupes_ == false) {
379  // Require dupes across the job.
380  BOOST_CHECK_GT(allEvents_.size(), uniqueEvents_.size());
382  // Require no dupes across the job.
383  BOOST_CHECK_EQUAL(allEvents_.size(), uniqueEvents_.size());
384  }
386  BOOST_CHECK_EQUAL(beginSubRunCounter_, 1ull);
387  BOOST_CHECK_EQUAL(endSubRunCounter_, 1ull);
388  BOOST_CHECK_EQUAL(beginRunCounter_, 1ull);
389  BOOST_CHECK_EQUAL(endRunCounter_, 1ull);
390 }
391 
392 #ifndef ART_TEST_NO_STARTEVENT
393 void
395 #ifdef ART_TEST_OLD_STARTEVENT
396  startEvent()
397 #else
398  // Normal case
400 #endif
401 {
402  startEvent_called_ = true;
403  eIDs_.reset();
404  ++currentEvent_;
405 }
406 #endif
407 
408 size_t
410 {
411  return (currentEvent_ == 2 && testZeroSecondaries_) ? 0 : nSecondaries_;
412 }
413 
414 void
416 {
417 #ifdef ART_TEST_NO_STARTEVENT
418  // Need to deal with this here
419  ++currentEvent_;
420 #endif
422  eIDs_ = std::make_unique<art::EventIDSequence>(seq);
423  if (!testEventOrdering_) {
424  return;
425  }
426  switch (readMode_) {
428  auto count(1);
429  for (auto const& eid : seq) {
430  BOOST_REQUIRE_EQUAL(eid.event(),
431  currentEvent_ * nSecondaries() + count++);
432  }
433  } break;
435  // We should have a duplicate within the secondaries.
436  std::unordered_set<int> s;
437  cet::transform_all(seq,
438  std::inserter(s, s.begin()),
439  [](art::EventID const& eid) { return eid.event(); });
440  BOOST_CHECK_GT(seq.size(), s.size());
441  } break;
443  if (testNoLimEventDupes_) {
444  // We should have no duplicate within the secondaries.
445  std::unordered_set<int> s;
447  seq, std::inserter(s, s.begin()), [](art::EventID const& eid) {
448  return eid.event() + eid.subRun() * 100 + (eid.run() - 1) * 500;
449  });
450  BOOST_CHECK_EQUAL(seq.size(), s.size());
451  } else { // Require dupes over 2 events.
452  auto checkpoint(allEvents_.size());
454  seq, std::back_inserter(allEvents_), [](art::EventID const& eid) {
455  return eid.event() + eid.subRun() * 100 + (eid.run() - 1) * 500;
456  });
457  uniqueEvents_.insert(allEvents_.cbegin() + checkpoint,
458  allEvents_.cend());
459  // Test at end job for duplicates.
460  }
461  break;
463  auto checkpoint(allEvents_.size());
465  seq, std::back_inserter(allEvents_), [](art::EventID const& eid) {
466  return eid.event() + eid.subRun() * 100 + (eid.run() - 1) * 500;
467  });
468  uniqueEvents_.insert(allEvents_.cbegin() + checkpoint, allEvents_.cend());
469  // Test at end job for no duplicates.
470  } break;
471  default:;
472  }
473 }
474 
475 void
477  art::EventAuxiliarySequence const& seq)
478 {
480  // We only need a very simple test to check that there actually
481  // are auxiliaries in the sequence. No need for full coverage,
482  // the event ids test takes care of that.
484  BOOST_REQUIRE_EQUAL(seq.size(), nSecondaries_);
485  size_t offset = 1;
486  for (auto const& val : seq) {
487  BOOST_REQUIRE_EQUAL(val.event(), offset);
488  ++offset;
489  }
490  }
491 }
492 
493 void
495 {
496  e.put(std::make_unique<std::string>("BlahBlahBlah"));
497  e.put(std::move(eIDs_));
498 #ifndef ART_TEST_NO_STARTEVENT
500  startEvent_called_ = false;
501 #endif
503  processEventIDs_called_ = false;
506 }
507 
508 void
510 {
512 }
513 
514 void
516 {
518 }
519 
520 void
522 {
524 }
525 
526 void
528 {
530 }
531 
532 void
534 {
536  subRunInfo_ = 0.0;
537 }
538 
539 void
541 {
543  sr.put(std::make_unique<double>(subRunInfo_));
544 }
545 
546 void
548 {
550  runInfo_ = 0.0;
551 }
552 
553 void
555 {
556  ++endRunCounter_;
557  r.put(std::make_unique<double>(runInfo_));
558 }
559 
560 template <typename PROD, typename OPROD>
561 bool
562 arttest::MixFilterTestDetail::mixByAddition(std::vector<PROD const*> const& in,
563  OPROD& out,
564  art::PtrRemapper const&)
565 {
566  verifyInSize(in);
567  for (auto const* prod : in) {
568  if (prod != nullptr) {
569  out += *prod;
570  }
571  }
572  return true; // Always want product in event.
573 }
574 
575 bool
577  std::vector<std::vector<double> const*> const& in,
578  std::vector<double>& out,
579  art::PtrRemapper const&)
580 {
581  verifyInSize(in);
583  return true; // Always want product in event.
584 }
585 
586 bool
588  std::vector<mv_t const*> const& in,
589  mv_t& out,
590  art::PtrRemapper const&)
591 {
592  verifyInSize(in);
594  return true; // Always want product in event.
595 }
596 
597 bool
599  std::vector<std::vector<art::Ptr<double>> const*> const& in,
601  art::PtrRemapper const& remap)
602 {
603  verifyInSize(in);
604  remap(in, std::back_inserter(out), doubleVectorOffsets_);
605  if (testPtrFailure_) {
606  BOOST_REQUIRE_THROW(*out.front(), art::Exception);
607  }
608  return true; // Always want product in event.
609 }
610 
611 #ifndef ART_NO_MIX_PTRVECTOR
612 bool
614  std::vector<art::PtrVector<double> const*> const& in,
616  art::PtrRemapper const& remap)
617 {
618  verifyInSize(in);
619  remap(in, std::back_inserter(out), doubleVectorOffsets_);
620  return true; // Always want product in event.
621 }
622 #endif
623 
624 bool
626  std::vector<arttest::ProductWithPtrs const*> const& in,
628  art::PtrRemapper const& remap)
629 {
630  verifyInSize(in);
631 #ifndef ART_NO_MIX_PTRVECTOR
632  remap(in,
633  std::back_inserter(out.ptrVectorDouble()),
636 #endif
637  remap(in,
638  std::back_inserter(out.vectorPtrDouble()),
641  // Throw-away object to test non-standard remap interface.
643  remap(in,
644  std::back_inserter(tmp.vectorPtrDouble()),
647  return true; // Always want product in event.
648 }
649 
650 bool
655  art::PtrRemapper const& remap)
656 {
657  verifyInSize(in);
658  remap(in, std::back_inserter(out), map_vectorOffsets_);
659  return true; // Always want product in event.
660 }
661 
662 bool
663 arttest::MixFilterTestDetail::mixSRProduct(std::vector<double const*> const& in,
664  DoubleProduct& out,
665  art::PtrRemapper const& remap)
666 {
667  mixByAddition(in, out, remap);
668  subRunInfo_ += out.value;
669  return true;
670 }
671 
672 bool
673 arttest::MixFilterTestDetail::mixRProduct(std::vector<double const*> const& in,
674  DoubleProduct& out,
675  art::PtrRemapper const& remap)
676 {
677  mixByAddition(in, out, remap);
678  runInfo_ += out.value;
679  return true;
680 }
681 
void startEvent(art::Event const &)
std::vector< EventAuxiliary > EventAuxiliarySequence
Definition: MixTypes.h:24
void respondToCloseInputFile(art::FileBlock const &fb)
std::vector< EventID > EventIDSequence
Definition: MixTypes.h:22
art::PtrVector< double > const & ptrVectorDouble() const
std::vector< art::Ptr< double > > const & vectorPtrDouble() const
bool mixPtrs(std::vector< std::vector< art::Ptr< double >> const * > const &in, std::vector< art::Ptr< double >> &out, art::PtrRemapper const &remap)
const char expected[]
Definition: Exception_t.cc:22
void respondToCloseOutputFiles(art::FileBlock const &fb)
std::string string
Definition: nybbler.cc:12
bool mixByAddition(std::vector< PROD const * > const &, OPROD &, art::PtrRemapper const &)
std::pair< key_type, mapped_type > value_type
Definition: map_vector.h:89
MixFilterTestDetail(Parameters const &p, art::MixHelper &helper)
bool mixPtrVectors(std::vector< art::PtrVector< double > const * > const &in, art::PtrVector< double > &out, art::PtrRemapper const &remap)
bool mixmap_vectorPtrs(std::vector< std::vector< art::Ptr< cet::map_vector< unsigned int >::value_type >> const * > const &in, std::vector< art::Ptr< cet::map_vector< unsigned int >::value_type >> &out, art::PtrRemapper const &remap)
void processEventAuxiliaries(art::EventAuxiliarySequence const &)
RunNumber_t run() const
Definition: EventID.h:99
Definition: Run.h:21
bool aggregateDoubleCollection(std::vector< std::vector< double > const * > const &in, std::vector< double > &out, art::PtrRemapper const &)
Value mapped_type
Definition: map_vector.h:88
auto vector(Vector const &v)
Returns a manipulator which will print the specified array.
Definition: DumpUtils.h:265
void beginRun(art::Run const &r)
void flattenCollections(std::vector< COLLECTION const * > const &in, COLLECTION &out)
const double e
#define DEFINE_ART_MODULE(klass)
Definition: ModuleMacros.h:68
void swap(Handle< T > &a, Handle< T > &b)
void verifyInSize(COLL const &in) const
parameter set interface
std::vector< size_t > map_vectorOffsets_
std::function< bool(std::vector< PROD const * > const &, OPROD &, PtrRemapper const &)> MixFunc
Definition: MixTypes.h:17
void beginSubRun(art::SubRun const &sr)
art::MixHelper::Mode const readMode_
auto transform_all(Container &, OutputIt, UnaryOp)
MixFilterTestDetail & operator=(MixFilterTestDetail const &)=delete
BOOST_REQUIRE(inFile)
bool mixRProduct(std::vector< double const * > const &in, DoubleProduct &out, art::PtrRemapper const &)
std::unique_ptr< art::EventIDSequence > eIDs_
std::vector< art::Ptr< double > > vpd_
std::unordered_set< int > uniqueEvents_
p
Definition: test.py:228
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
bool aggregate_map_vector(std::vector< mv_t const * > const &in, mv_t &out, art::PtrRemapper const &)
void respondToOpenInputFile(art::FileBlock const &fb)
std::vector< size_t > doubleVectorOffsets_
T copy(T const &v)
bool mixSRProduct(std::vector< double const * > const &in, DoubleProduct &out, art::PtrRemapper const &)
void respondToOpenOutputFiles(art::FileBlock const &fb)
EventNumber_t event() const
Definition: EventID.h:117
static const double s
Definition: Units.h:99
bool mixProductWithPtrs(std::vector< arttest::ProductWithPtrs const * > const &in, arttest::ProductWithPtrs &out, art::PtrRemapper const &remap)
ProductID put(std::unique_ptr< PROD > &&edp, FullSemantic< Level::Run > const semantic)
Definition: DataViewImpl.h:692
static const double sr
Definition: Units.h:167
SubRunNumber_t subRun() const
Definition: EventID.h:111
cet::coded_exception< error, detail::translate > exception
Definition: exception.h:33
void processEventIDs(art::EventIDSequence const &seq)