MixHelper.h
Go to the documentation of this file.
1 #ifndef art_Framework_IO_ProductMix_MixHelper_h
2 #define art_Framework_IO_ProductMix_MixHelper_h
3 ////////////////////////////////////////////////////////////////////////
4 // MixHelper
5 //
6 // Class providing, at construction time, registration services to
7 // users' "detail" mixing classes. (A "detail" class is the template
8 // argument to the instantiation of the "MixFilter" module template.)
9 //
10 ////////////////////////////////////////////////////////////////////////
11 // Configuration.
12 //
13 // MixHelper will be passed the configuration of the module. The
14 // following items are significant:
15 //
16 // fileNames (default empty)
17 //
18 // Sequence of secondary files for mixing. However, see the function
19 // registerSecondaryFileNameProvider(...) below. If a secondary file
20 // name provider is *not* registered, it is an error to have an
21 // empty fileNames.
22 //
23 // readMode (default sequential).
24 //
25 // Specify how events should be chosen from each file. Valid values
26 // are:
27 //
28 // sequential -- read the secondary events in order
29 // randomReplace -- random with replacement
30 // randomLimReplace -- events unique within a primary event
31 // randomNoReplace -- events guaranteed to be used once only.
32 //
33 // MT note: What gets mixed is unpredictable when multiple mixing
34 // modules run in parallel either because of multiple trigger
35 // paths or because of streams when the mixing mode is
36 // Mode::SEQUENTIAL. For art 3.0, mixing will be entirely
37 // serialized.
38 //
39 // coverageFraction (default 1.0).
40 //
41 // Ratio of sampled to total events in a file. Used by randomReplace
42 // and randomLimReplace modes only.
43 //
44 // wrapFiles (default false).
45 //
46 // Re-start from fileNames[0] after secondary events are
47 // exhausted. If this is false, exhausting the secondary file stream
48 // will result in the filter throwing an exception that is handled
49 // based on the configured handling policy for FileReadError
50 // exceptions.
51 //
52 // compactMissingProducts (default false).
53 //
54 // In the case of a secondary event missing a particular product,
55 // the sequence of product pointers passed to the MixOp will be
56 // compacted to remove nullptrs.
57 //
58 ////////////////////////////////////////////////////////////////////////
59 // readMode()
60 //
61 // Return the enumerated value representing the event mixing strategy.
62 //
63 ////////////////////////////////////////////////////////////////////////
64 // registerSecondaryFileNameProvider(<function> func)
65 //
66 // Register the provided function as a provider of file names for
67 // mixing. This should be called from the constructor of your detail
68 // object.
69 //
70 // <function> must be convertible to std::function<std::string()>. A
71 // free function taking no arguments and returning std::string, a
72 // functor whose operator() has the same signature, or a bound free or
73 // member function whose signature after binding is std::string() are
74 // all convertible to std::function<std::string()>.
75 //
76 // E.g. for a detail class with member function std::string getMixFile():
77 //
78 // registerSecondaryFileNameProvider(std::bind(&Detail::getMixFile, this));
79 //
80 // Notes:
81 //
82 // 1. It is a configuration error to provide a non-empty fileNames
83 // parameter to a module which registers a file name provider.
84 //
85 // 2. If the file name provider returns a string which is empty, the
86 // MixFilter shall thenceforth return false.
87 //
88 // 3. If the file name provider returns a non-empty string that does
89 // not correspond to a readable file, an exception shall be thrown.
90 //
91 ////////////////////////////////////////////////////////////////////////
92 // declareMixOp templates.
93 //
94 // These function templates should be used by writers of
95 // product-mixing "detail" classes to declare each product mix
96 // operation.
97 //
98 // All of the declareMixOp(...) function templates have the following
99 // template arguments:
100 //
101 // 1. The BranchType (defaults to art::InEvent). Specify explicitly if
102 // you wish to mix subrun or run products, or if you need to
103 // specify explicitly any of the other template arguments.
104 //
105 // 2. The incoming product type (deduced from the provided callable
106 // mixer argument).
107 //
108 // 3. The outgoing product type (deduced from the provided callable
109 // mixer argument).
110 //
111 // A product mixing operation should be specified by providing:
112 //
113 // 1. an InputTag specifying which secondary products should be mixed;
114 //
115 // 2. an optional instance label for the mixed product (defaulting to
116 // the instance label of the incoming product if unspecified); and
117 //
118 // 3. a callable mixer such as:
119 //
120 // bool mixfunc(std::vector<PROD const*> const&,
121 // OPROD&,
122 // PtrRemapper const&),
123 //
124 // As the user may prefer, the mixer may take the form of:
125 // a) an arbitrarily-named free function, or
126 // b) a function object whose operator() must have a signature
127 // whose arguments match those above, or
128 // c) an arbitrarily-named member function of any class.
129 // In each case, the mixer must have the same type (i.e.,
130 // the same return type and the same parameter types) illustrated
131 // by "mixfunc" above. The return value of the mix function is
132 // taken to indicate whether the product should be placed in the
133 // event.
134 //
135 // Generally speaking the PROD and OPROD template arguments are
136 // deducible from the callable mixer function which, once bound
137 // (if appropriate) to any provided class instance, should be
138 // convertible to an instantiation of the template instance
139 // art::MixFunc<PROD, OPROD>. See
140 // art/Framework/IO/ProductMix/MixTypes.h for more details.
141 //
142 // If the provided function is a member function it may be
143 // provided bound to the object upon which it is to be called by
144 // the user (in which case it is treated as a free function by the
145 // registration method) or by specifying the member function
146 // followed by the object to which it should be bound (in which
147 // case the bind will be done for the user). In this latter case
148 // the template argument specifying the product type need *not* be
149 // specified usually as it may be deduced from the signature of
150 // the provided function. If one specifies an overload set however
151 // (e.g. in the case where a class has several mix() member
152 // functions, each one with a different mix function) then the
153 // template argument must be specified in order to constrain the
154 // overload set to a single function.
155 //
156 // See also the description of the template alias
157 // art::MixFunc<PROD, OPROD> defined in
158 // art/Framework/IO/ProductMix/MixTypes.h.
159 //
160 // 4. An optional boolean, "outputProduct," defaulting to, "true." A
161 // false value for this parameter indicates that the mix product
162 // will *never* be put into the event and should therefore not be
163 // declared. If the mix operation so declared ever returns true an
164 // exception will be thrown.
165 //
166 // declareMixOp() may be called with any of the following argument
167 // combinations:
168 //
169 // 1. Provide an InputTag and a mixer that is a free function or
170 // function object (wrapped as a MixFunc<PROD, OPROD>).
171 //
172 // 2. Provide an InputTag, an output instance label, and a mixer
173 // that is a free function or function object.
174 //
175 // 3. Provide an InputTag, a mixer that is a non-const member
176 // function (of any class), and an object to which that member
177 // function should be bound.
178 //
179 // 4. Provide an InputTag, an output instance label, a mixer that is
180 // a non-const member function (of any class), and an object to
181 // which that member function should be bound.
182 //
183 // 5. Same as 3, but providing a mixer that is a const member function.
184 //
185 // 6. Same as 4, but providing a mixer that is a const member function.
186 //
187 // Note: For signatures 3-6, if the compiler complains about an
188 // unresolved overload your first move should be to specify the
189 // product type(s) as template argument (to do this you must specify
190 // explicitly the BranchType template argument first). If that does
191 // not resolve the problem, try an explicit:
192 //
193 // const_cast<T const&>(t)
194 //
195 // or
196 //
197 // const_cast<T&>(t)
198 //
199 // as appropriate.
200 //
201 ////////////////////////////////////
202 // produces() templates.
203 //
204 // Call as you would from the constructor of a module to declare
205 // (e.g., bookkeeping) products to be put into the event that are
206 // *not* direct results of a single product mix operation. For the
207 // latter case, see the declareMixOp() templates above.
208 //
209 // Signatures for produces():
210 //
211 // 1. produces<PROD>(optional_instance_name);
212 //
213 // Register a product to go into the event.
214 //
215 // 2. produces<PROD, art::InRun>(optional_instance_name);
216 // produces<PROD, art::InSubRun>(optional_instance_name);
217 //
218 // Register a product to go into the run or subrun.
219 //
220 ////////////////////////////////////////////////////////////////////////
221 
222 #include "CLHEP/Random/RandFlat.h"
232 #include "cetlib/exempt_ptr.h"
233 #include "fhiclcpp/fwd.h"
234 #include "fhiclcpp/types/Atom.h"
235 #include "fhiclcpp/types/Sequence.h"
236 
237 #include <functional>
238 #include <iosfwd>
239 #include <memory>
240 #include <string>
241 #include <vector>
242 
243 namespace art {
244 
246  using ProviderFunc_ = std::function<std::string()>;
247 
248  public:
249  enum class Mode {
250  SEQUENTIAL = 0,
254  UNKNOWN
255  };
256 
257  struct Config {
258  fhicl::Sequence<std::string> filenames{fhicl::Name{"fileNames"}, {}};
259  fhicl::Atom<bool> compactMissingProducts{
260  fhicl::Name{"compactMissingProducts"},
261  false};
262  fhicl::Atom<std::string> readMode{fhicl::Name{"readMode"}, "sequential"};
263  fhicl::Atom<double> coverageFraction{fhicl::Name{"coverageFraction"},
264  1.0};
265  fhicl::Atom<bool> wrapFiles{fhicl::Name{"wrapFiles"}, false};
266  fhicl::Atom<seed_t> seed{fhicl::Name{"seed"}, -1};
267  };
268 
269  explicit MixHelper(Config const& config,
270  std::string const& moduleLabel,
271  ProducesCollector& collector,
272  std::unique_ptr<MixIOPolicy> ioHandle);
273  explicit MixHelper(fhicl::ParameterSet const& pset,
274  std::string const& moduleLabel,
275  ProducesCollector& collector,
276  std::unique_ptr<MixIOPolicy> ioHandle);
277 
278  // Returns the current mixing mode.
279  Mode readMode() const;
280 
281  // Registers a callback to the detail object to determine the next
282  // secondary file to read.
284 
285  // A.
286  template <class P>
287  void produces(std::string const& instanceName = {});
288 
289  // B.
290  template <class P, BranchType B>
291  void produces(std::string const& instanceName = {});
292 
293  // 1.
294  template <art::BranchType B = art::InEvent, typename PROD, typename OPROD>
295  void declareMixOp(InputTag const& inputTag,
296  MixFunc<PROD, OPROD> mixFunc,
297  bool outputProduct = true);
298 
299  // 2.
300  template <art::BranchType B = art::InEvent, typename PROD, typename OPROD>
301  void declareMixOp(InputTag const& inputTag,
302  std::string const& outputInstanceLabel,
303  MixFunc<PROD, OPROD> mixFunc,
304  bool outputProduct = true);
305 
306  // 3.
307  template <art::BranchType B = art::InEvent,
308  typename PROD,
309  typename OPROD,
310  typename T>
311  void declareMixOp(InputTag const& inputTag,
312  bool (T::*mixfunc)(std::vector<PROD const*> const&,
313  OPROD&,
314  PtrRemapper const&),
315  T& t,
316  bool outputProduct = true);
317 
318  // 4.
319  template <art::BranchType B = art::InEvent,
320  typename PROD,
321  typename OPROD,
322  typename T>
323  void declareMixOp(InputTag const& inputTag,
324  std::string const& outputInstanceLabel,
325  bool (T::*mixfunc)(std::vector<PROD const*> const&,
326  OPROD&,
327  PtrRemapper const&),
328  T& t,
329  bool outputProduct = true);
330 
331  // 5.
332  template <art::BranchType B = art::InEvent,
333  typename PROD,
334  typename OPROD,
335  typename T>
336  void declareMixOp(InputTag const& inputTag,
337  bool (T::*mixfunc)(std::vector<PROD const*> const&,
338  OPROD&,
339  PtrRemapper const&) const,
340  T const& t,
341  bool outputProduct = true);
342 
343  // 6.
344  template <art::BranchType B = art::InEvent,
345  typename PROD,
346  typename OPROD,
347  typename T>
348  void declareMixOp(InputTag const& inputTag,
349  std::string const& outputInstanceLabel,
350  bool (T::*mixfunc)(std::vector<PROD const*> const&,
351  OPROD&,
352  PtrRemapper const&) const,
353  T const& t,
354  bool outputProduct = true);
355 
356  // Random number engine creation
359  std::string const& kind_of_engine_to_make);
361  std::string const& kind_of_engine_to_make,
362  label_t const& engine_label);
363 
364  //////////////////////////////////////////////////////////////////////
365  // Mix module writers should not need anything below this point.
366  //////////////////////////////////////////////////////////////////////
367  bool generateEventSequence(size_t nSecondaries,
368  EntryNumberSequence& enSeq,
369  EventIDSequence& eIDseq);
371  EntryNumberSequence const&);
372  void mixAndPut(EntryNumberSequence const& enSeq,
373  EventIDSequence const& eIDseq,
374  Event& e);
375  void setEventsToSkipFunction(std::function<size_t()> eventsToSkip);
376 
377  private:
378  MixHelper(MixHelper const&) = delete;
379  MixHelper& operator=(MixHelper const&) = delete;
380 
381  using MixOpList = std::vector<std::unique_ptr<MixOpBase>>;
382 
384  std::unique_ptr<CLHEP::RandFlat> initDist_(
385  cet::exempt_ptr<base_engine_t> engine) const;
386  bool consistentRequest_(std::string const& kind_of_engine_to_make,
387  label_t const& engine_label) const;
388  Mode initReadMode_(std::string const& mode) const;
389  bool openNextFile_();
390 
392  MixOpList& mixOps);
393 
396  std::vector<std::string> const filenames_;
403  double const coverageFraction_;
404  std::size_t nEventsReadThisFile_{};
405  std::size_t totalEventsRead_{};
406  bool const canWrapFiles_;
410  std::unique_ptr<CLHEP::RandFlat> dist_;
411  std::function<size_t()> eventsToSkip_{};
412  EntryNumberSequence shuffledSequence_{}; // RANDOM_NO_REPLACE only.
413  bool haveSubRunMixOps_{false};
414  bool haveRunMixOps_{false};
416 
417  std::unique_ptr<MixIOPolicy> ioHandle_{nullptr};
418  };
419 
420  std::ostream& operator<<(std::ostream&, MixHelper::Mode);
421 }
422 
423 inline auto
425 {
426  return readMode_;
427 }
428 
429 // A.
430 template <class P>
431 inline void
433 {
434  collector_.produces<P>(instanceName);
435 }
436 
437 // B.
438 template <class P, art::BranchType B>
439 inline void
441 {
442  collector_.produces<P, B>(instanceName);
443 }
444 
445 // 1.
446 template <art::BranchType B, typename PROD, typename OPROD>
447 inline void
449  MixFunc<PROD, OPROD> mixFunc,
450  bool outputProduct)
451 {
452  declareMixOp<B>(inputTag, inputTag.instance(), mixFunc, outputProduct); // 2.
453 }
454 
455 // 2.
456 template <art::BranchType B, typename PROD, typename OPROD>
457 void
459  std::string const& outputInstanceLabel,
460  MixFunc<PROD, OPROD> mixFunc,
461  bool outputProduct)
462 {
463  if (outputProduct) {
464  produces<OPROD>(outputInstanceLabel);
465  }
466  if (B == art::InSubRun) {
467  haveSubRunMixOps_ = true;
468  } else if (B == art::InRun) {
469  haveRunMixOps_ = true;
470  }
471  mixOps_.emplace_back(new MixOp<PROD, OPROD>(moduleLabel_,
472  inputTag,
473  outputInstanceLabel,
474  mixFunc,
475  outputProduct,
477  B));
478 }
479 
480 // 3.
481 template <art::BranchType B, typename PROD, typename OPROD, typename T>
482 inline void
484  bool (T::*mixFunc)(std::vector<PROD const*> const&,
485  OPROD&,
486  PtrRemapper const&),
487  T& t,
488  bool outputProduct)
489 {
490  declareMixOp<B>(inputTag,
491  inputTag.instance(),
492  mixFunc,
493  t,
494  outputProduct); // 4.
495 }
496 
497 // 4.
498 template <art::BranchType B, typename PROD, typename OPROD, typename T>
499 void
501  std::string const& outputInstanceLabel,
502  bool (T::*mixFunc)(std::vector<PROD const*> const&,
503  OPROD&,
504  PtrRemapper const&),
505  T& t,
506  bool outputProduct)
507 {
508  using std::placeholders::_1;
509  using std::placeholders::_2;
510  using std::placeholders::_3;
511  declareMixOp<B>(
512  inputTag,
513  outputInstanceLabel,
514  static_cast<MixFunc<PROD, OPROD>>(std::bind(mixFunc, &t, _1, _2, _3)),
515  outputProduct); // 2.
516 }
517 
518 // 5.
519 template <art::BranchType B, typename PROD, typename OPROD, typename T>
520 inline void
522  bool (T::*mixFunc)(std::vector<PROD const*> const&,
523  OPROD&,
524  PtrRemapper const&) const,
525  T const& t,
526  bool outputProduct)
527 {
528  declareMixOp<B>(inputTag,
529  inputTag.instance(),
530  mixFunc,
531  t,
532  outputProduct); // 6.
533 }
534 
535 // 6.
536 template <art::BranchType B, typename PROD, typename OPROD, typename T>
537 void
539  std::string const& outputInstanceLabel,
540  bool (T::*mixFunc)(std::vector<PROD const*> const&,
541  OPROD&,
542  PtrRemapper const&) const,
543  T const& t,
544  bool outputProduct)
545 {
546  using std::placeholders::_1;
547  using std::placeholders::_2;
548  using std::placeholders::_3;
549  declareMixOp<B>(
550  inputTag,
551  outputInstanceLabel,
552  static_cast<MixFunc<PROD, OPROD>>(std::bind(mixFunc, &t, _1, _2, _3)),
553  outputProduct); // 2.
554 }
555 
556 #endif /* art_Framework_IO_ProductMix_MixHelper_h */
557 
558 // Local Variables:
559 // mode: c++
560 // End:
void registerSecondaryFileNameProvider(ProviderFunc_ func)
Definition: MixHelper.cc:137
std::vector< std::string >::const_iterator fileIter_
Definition: MixHelper.h:401
std::vector< EventAuxiliary > EventAuxiliarySequence
Definition: MixTypes.h:28
std::function< size_t()> eventsToSkip_
Definition: MixHelper.h:411
std::vector< EventID > EventIDSequence
Definition: MixTypes.h:26
ProviderFunc_ providerFunc_
Definition: MixHelper.h:398
Mode initReadMode_(std::string const &mode) const
Definition: MixHelper.cc:357
cet::exempt_ptr< base_engine_t > initEngine_(seed_t seed, Mode readMode)
Definition: MixHelper.cc:454
void mixAndPut(EntryNumberSequence const &enSeq, EventIDSequence const &eIDseq, Event &e)
Definition: MixHelper.cc:278
void setEventsToSkipFunction(std::function< size_t()> eventsToSkip)
Definition: MixHelper.cc:351
bool consistentRequest_(std::string const &kind_of_engine_to_make, label_t const &engine_label) const
Definition: MixHelper.cc:432
std::size_t totalEventsRead_
Definition: MixHelper.h:405
std::string string
Definition: nybbler.cc:12
RNGsnapshot::label_t label_t
Definition: EngineCreator.h:37
std::map< ProductID, ProductID > ProductIDTransMap
std::vector< std::unique_ptr< MixOpBase >> MixOpList
Definition: MixHelper.h:381
MixOpList mixOps_
Definition: MixHelper.h:399
bool haveSubRunMixOps_
Definition: MixHelper.h:413
std::string const & instance() const noexcept
Definition: InputTag.cc:85
std::pair< float, std::string > P
unsigned nOpensOverThreshold_
Definition: MixHelper.h:407
EntryNumberSequence shuffledSequence_
Definition: MixHelper.h:412
intermediate_table::const_iterator const_iterator
std::vector< std::string > const filenames_
Definition: MixHelper.h:396
void produces(std::string const &instanceName={})
Definition: MixHelper.h:432
base_engine_t & createEngine(seed_t seed)
Definition: MixHelper.cc:148
std::unique_ptr< CLHEP::RandFlat > dist_
Definition: MixHelper.h:410
double const coverageFraction_
Definition: MixHelper.h:403
Mode const readMode_
Definition: MixHelper.h:402
bool generateEventSequence(size_t nSecondaries, EntryNumberSequence &enSeq, EventIDSequence &eIDseq)
Definition: MixHelper.cc:182
std::ostream & operator<<(std::ostream &os, const GroupSelector &gs)
bool const canWrapFiles_
Definition: MixHelper.h:406
void produces(std::string const &instanceName={}, Persistable const persistable=Persistable::Yes)
const double e
void declareMixOp(InputTag const &inputTag, MixFunc< PROD, OPROD > mixFunc, bool outputProduct=true)
Definition: MixHelper.h:448
ProducesCollector & collector_
Definition: MixHelper.h:394
static Config * config
Definition: config.cpp:1054
bool openNextFile_()
Definition: MixHelper.cc:384
bool compactMissingProducts_
Definition: MixHelper.h:397
std::unique_ptr< MixIOPolicy > ioHandle_
Definition: MixHelper.h:417
std::function< bool(std::vector< PROD const * > const &, OPROD &, PtrRemapper const &)> MixFunc
Definition: MixTypes.h:19
MixHelper(Config const &config, std::string const &moduleLabel, ProducesCollector &collector, std::unique_ptr< MixIOPolicy > ioHandle)
Definition: MixHelper.cc:99
CLHEP::HepRandomEngine base_engine_t
Definition: EngineCreator.h:36
std::string const moduleLabel_
Definition: MixHelper.h:395
std::vector< FileIndex::EntryNumber_t > EntryNumberSequence
Definition: MixTypes.h:27
cet::exempt_ptr< base_engine_t > engine_
Definition: MixHelper.h:409
PtrRemapper ptrRemapper_
Definition: MixHelper.h:400
std::size_t nEventsReadThisFile_
Definition: MixHelper.h:404
std::function< std::string()> ProviderFunc_
Definition: MixHelper.h:246
std::map< FileIndex::EntryNumber_t, EventID > EventIDIndex
Definition: MixTypes.h:23
Mode readMode() const
Definition: MixHelper.h:424
bool haveRunMixOps_
Definition: MixHelper.h:414
MixHelper & operator=(MixHelper const &)=delete
EventAuxiliarySequence generateEventAuxiliarySequence(EntryNumberSequence const &)
Definition: MixHelper.cc:268
EventIDIndex eventIDIndex_
Definition: MixHelper.h:415
ProdToProdMapBuilder::ProductIDTransMap buildProductIDTransMap_(MixOpList &mixOps)
def func()
Definition: docstring.py:7
BranchType
Definition: BranchType.h:20
std::unique_ptr< CLHEP::RandFlat > initDist_(cet::exempt_ptr< base_engine_t > engine) const
Definition: MixHelper.cc:472
void function(int client, int *resource, int parblock, int *test, int p)
ProdToProdMapBuilder ptpBuilder_
Definition: MixHelper.h:408