cvnCreateZlibImages.cc
Go to the documentation of this file.
1 ////////////////////////////////////////////////////////////////////////////////////////////////////
2 //// Authors: Saul Alonso-Monsalve, saul.alonso.monsalve@cern.ch
3 //// Zlib image generator.
4 ////
5 //////////////////////////////////////////////////////////////////////////////////////////////////////
6 
7 // std library
8 #include <iostream>
9 #include <sys/stat.h>
10 #include <fstream>
11 #include <sstream>
12 #include <algorithm>
13 
14 // Boost, for program options
15 #include "boost/program_options/options_description.hpp"
16 #include "boost/program_options/variables_map.hpp"
17 #include "boost/program_options/parsers.hpp"
18 #include "boost/algorithm/string/predicate.hpp"
19 
20 // ART/fcl stuff
21 #include "cetlib/filepath_maker.h"
24 #include "fhiclcpp/ParameterSet.h"
25 
26 // ROOT stuff
27 #include "TChain.h"
28 #include "TFile.h"
29 
30 // CVN stuff
32 //#include "CVN/art/CaffeNetHandler.h"
33 
34 #include "zlib.h" // compression algorithm
35 
36 namespace po = boost::program_options;
37 
38 class Config
39 {
40 public:
42  fTreeName (pset.get<std::string>("TreeName")),
43  fOutputDir (pset.get<std::string>("OutputDir")),
44  fSetLog (pset.get<bool>("SetLog")),
45  fNEvents (pset.get<unsigned int>("NEvents")),
46  fPlaneLimit (pset.get<unsigned int>("PlaneLimit")),
47  fTDCLimit (pset.get<unsigned int>("TDCLimit")),
48  fReverseViews(pset.get<std::vector<bool> >("ReverseViews"))
49  {
50  };
51 
55 
56  /// Number of training examples for each test sample, e.g. 4 for 80/20 split
57  /// Number of examples in between progress updates (% complete)
58 
59  bool fSetLog;
60 
61  /// Flag to control whether or not we write HDF5 regression features
62  /// Limit the number of entries in the tree to consider
63  unsigned int fNEvents;
64  /// Limit the number of wires in the output image
65  int fPlaneLimit;
66  /// Limit the number of TDCs in the output image
67  int fTDCLimit;
68  /// Views to reverse
69  std::vector<bool> fReverseViews;
70 };
71 
73 {
74 
75  TChain chain(config.fTreeName.c_str());
76 
77  if (boost::ends_with(input,".list")) {
78  std::ifstream list_file(input.c_str());
79  if (!list_file.is_open()) {
80  std::cout << "Could not open " << input << std::endl;
81  exit(1);
82  }
83 
84  std::string ifname;
85  while (list_file>>ifname)
86  chain.Add(ifname.c_str());
87 
88  }//end if list file
89 
90  else if (boost::ends_with(input,".root")) {
91  chain.Add(input.c_str());
92  }//end if root file
93 
94  chain.SetMakeClass(1);
95 
96  int fInt;
97  UInt_t fPMap_fNWire;
98  UInt_t fPMap_fNTdc;
99  std::vector<float> fPMap_fPEX;
100  std::vector<float> fPMap_fPEY;
101  std::vector<float> fPMap_fPEZ;
102 
103  chain.SetBranchAddress("fInt", &fInt);
104  chain.SetBranchAddress("fPMap.fNWire", &fPMap_fNWire);
105  chain.SetBranchAddress("fPMap.fNTdc", &fPMap_fNTdc);
106  chain.SetBranchAddress("fPMap.fPEX", &fPMap_fPEX);
107  chain.SetBranchAddress("fPMap.fPEY", &fPMap_fPEY);
108  chain.SetBranchAddress("fPMap.fPEZ", &fPMap_fPEZ);
109 
110  float fNuEnergy = -1;
111  float fLepEnergy = -1;
112  float fRecoNueEnergy = -1;
113  float fRecoNumuEnergy = -1;
114  float fEventWeight = -1;
115 
116  int fNuPDG = -1;
117  int fNProton = -1;
118  int fNPion = -1;
119  int fNPizero = -1;
120  int fNNeutron = -1;
121 
122  int fTopologyType = -1;
123  int fTopologyTypeAlt = -1;
124 
125  chain.SetBranchAddress("fNuEnergy", &fNuEnergy);
126  chain.SetBranchAddress("fLepEnergy", &fLepEnergy);
127  chain.SetBranchAddress("fRecoNueEnergy", &fRecoNueEnergy);
128  chain.SetBranchAddress("fRecoNumuEnergy", &fRecoNumuEnergy);
129  chain.SetBranchAddress("fEventWeight", &fEventWeight);
130 
131  chain.SetBranchAddress("fNuPDG", &fNuPDG);
132  chain.SetBranchAddress("fNProton", &fNProton);
133  chain.SetBranchAddress("fNPion", &fNPion);
134  chain.SetBranchAddress("fNPizero", &fNPizero);
135  chain.SetBranchAddress("fNNeutron", &fNNeutron);
136 
137  chain.SetBranchAddress("fTopologyType", &fTopologyType);
138  chain.SetBranchAddress("fTopologyTypeAlt", &fTopologyTypeAlt);
139 
140  unsigned int entries = chain.GetEntries();
141  if(config.fNEvents < entries){
142  entries = config.fNEvents;
143  }
144  if(entries <= 0){
145  std::cout << "Error: Input tree has no entries." << std::endl;
146  exit(4);
147  }
148 
149  std::cout << "- Will process " << entries << " from the input tree." << std::endl;
150 
151  std::srand ( unsigned ( std::time(0) ) );
152  std::vector<unsigned int> shuffled;
153  for (unsigned int i = 0; i < entries; ++i)
154  {
155  shuffled.push_back(i);
156  }
157 
158  if(entries > chain.GetEntries()){
159  entries = chain.GetEntries();
160  }
161 
162  std::string image_path = config.fOutputDir + "/images/";
163  std::string info_path = config.fOutputDir + "/info/";
164 
165  // create image path
166  if (mkdir(image_path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == -1)
167  {
168  if( errno == EEXIST ) {
169  // alredy exists
170  }else{
171  // error
172  std::cout << "cannot create folder error:" << strerror(errno) << std::endl;
173  exit(1);
174  }
175  }
176  // create info path
177  if (mkdir(info_path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == -1)
178  {
179  if( errno == EEXIST ) {
180  // alredy exists
181  }else{
182  // error
183  std::cout << "cannot create folder error:" << strerror(errno) << std::endl;
184  exit(1);
185  }
186  }
187 
188  for(unsigned int iEntry = 0; iEntry < entries; ++iEntry)
189  {
190  unsigned int entry = shuffled[iEntry];
191  chain.GetEntry(entry);
192 
193  unsigned int nViews = 3;
194 
195  // Create a CVNImageUtils object and use it to produce the pixels. The arguments
196  // define how large we want the output image to be
197  cvn::CVNImageUtils imageUtils(config.fPlaneLimit,config.fTDCLimit,nViews);
198  // Since we don't have a PixelMap object, we need to tell it how big it is
199  imageUtils.SetPixelMapSize(fPMap_fNWire,fPMap_fNTdc);
200 
201  std::vector<unsigned char> pixelArray(nViews * config.fPlaneLimit * config.fTDCLimit,0);
202 
203  imageUtils.SetLogScale(config.fSetLog);
204  imageUtils.SetViewReversal(config.fReverseViews);
205  imageUtils.ConvertChargeVectorsToPixelArray(fPMap_fPEX, fPMap_fPEY, fPMap_fPEZ, pixelArray);
206 
207  //std::cout << "fNuEnergyi: " << fNuEnergy << std::endl;
208  //std::cout << "fRecoNueEnergy: " << fRecoNueEnergy << std::endl;
209  //std::cout << "fRecoNumuEnergy: " << fRecoNumuEnergy << std::endl;
210  //std::cout << "fEventWeight: " << fEventWeight << std::endl;
211 
212  std::cout << "[DEBUG] entry " << entry+1 << " out of " << entries << std::endl;
213  std::cout << "[DEBUG] file: " << chain.GetCurrentFile()->GetName() << std::endl;
214  //std::cout << "[DEBUG] channels: " << nViews << std::endl;
215  //std::cout << "[DEBUG] planes: " << config.fPlaneLimit << std::endl;
216  //std::cout << "[DEBUG] cells: " << config.fTDCLimit << std::endl;
217  //std::cout << "[DEBUG] channels*planes*cells: " << channels*planes*cells << std::endl;
218 
219  std::cout << "[DEBUG] label: " << fInt << std::endl;
220  unsigned long srcLen = nViews * config.fPlaneLimit * config.fTDCLimit; // pixelArray length
221  unsigned long destLen = compressBound(srcLen); // calculate size of the compressed data
222  char* ostream = (char *) malloc(destLen); // allocate memory for the compressed data
223 
224  int res = compress((Bytef *) ostream, &destLen, (Bytef *) &pixelArray[0], srcLen); // compress pixels
225 
226  // destLen is now the size of actuall buffer needed for compression
227  // you don't want to uncompress whole buffer later, just the used part
228  //
229 
230  // Buffer error
231 
232  if(res == Z_BUF_ERROR){
233  std::cout << "[DEBUG] Buffer was too small!" << std::endl;
234  }
235 
236  // Memory error
237 
238  else if(res == Z_MEM_ERROR){
239  std::cout << "[DEBUG] Not enough memory for compression!" << std::endl;
240  }
241 
242  // Compression ok
243 
244  else{
245  std::cout << "[DEBUG] Compression successful" << std::endl;
246 
247  // Create output files
248 
249  std::string image_file_name = image_path + input + std::to_string(iEntry) + ".gz";
250  std::string info_file_name = info_path + input + std::to_string(iEntry) + ".info";
251 
252  while(1){
253 
254  std::ofstream image_file (image_file_name, std::ofstream::binary);
255  std::ofstream info_file (info_file_name);
256 
257  if(image_file.is_open() && info_file.is_open()){
258 
259  // Write compressed data to file
260 
261  image_file.write(ostream, destLen);
262 
263  image_file.close(); // close file
264 
265  // Write records to file
266 
267  // Category
268 
269  info_file << fInt << std::endl;
270 
271  // Energy
272 
273  info_file << fNuEnergy << std::endl;
274  info_file << fLepEnergy << std::endl;
275  info_file << fRecoNueEnergy << std::endl;
276  info_file << fRecoNumuEnergy << std::endl;
277  info_file << fEventWeight << std::endl;
278 
279  // Topology
280 
281  info_file << fNuPDG << std::endl;
282  info_file << fNProton << std::endl;
283  info_file << fNPion << std::endl;
284  info_file << fNPizero << std::endl;
285  info_file << fNNeutron << std::endl;
286 
287  info_file << fTopologyType << std::endl;
288  info_file << fTopologyTypeAlt;
289 
290  info_file.close(); // close file
291 
292  std::cout << "[DEBUG] Done" << std::endl;
293  break;
294 
295  }
296  else{
297 
298  if(image_file.is_open()){
299  image_file.close(); // close file
300  }
301  else{
302  std::cout << "[DEBUG] Unable to open file: " << image_file_name << std::endl;
303  }
304 
305  if(info_file.is_open()){
306  info_file.close(); // close file
307  }
308  else{
309  std::cout << "[DEBUG] Unable to open file: " << info_file_name << std::endl;
310  }
311  }
312  }
313  //else std::cout << "[DEBUG] Unable to open files " << image_file_name << " and " << info_file_name << std::endl;
314  }
315 
316  free(ostream); // free allocated memory
317 
318  }
319 }
320 
321 
322 po::variables_map getOptions(int argc, char* argv[], std::string& config,
324 {
325 
326  // Declare the supported options.
327  po::options_description desc("Allowed options");
328  desc.add_options()
329  ("help", "produce help message")
330  ("config,c", po::value<std::string>(&config)->required(),
331  "configuration file")
332  ("input,i", po::value<std::string>(&input)->required(),
333  "Input data in ROOT file.");
334  po::variables_map vm;
335 
336  try
337  {
338  po::store(po::parse_command_line(argc, argv, desc), vm);
339  po::notify(vm);
340 
341  }
342  catch(po::error& e)
343  {
344  std::cout << "ERROR: " << e.what() << std::endl;
345  exit(1);
346  }
347 
348 
349  if (vm.count("help")) {
350  std::cout << desc << "\n";
351  exit(1);
352  }
353 
354  return vm;
355 }
356 
357 
358 
360 {
361  cet::filepath_first_absolute_or_lookup_with_dot policy("FHICL_FILE_PATH");
362  return fhicl::ParameterSet::make(fhicl::parse_document(configPath, policy));
363 }
364 
365 int main(int argc, char* argv[])
366 {
367 
368  std::string configPath, inputPath, outputPath, logPath;
369  po::variables_map vm = getOptions(argc, argv, configPath, inputPath);
370 
371  Config config(getPSet(configPath));
372 
373 
374  fill(config, inputPath);
375 
376  return 0;
377 
378 }
379 
380 
381 
void fill(const Config &config, std::string input)
Config(const fhicl::ParameterSet &pset)
QList< Entry > entry
bool fSetLog
Definition: cvnCreateDB.cc:108
def mkdir(path, mode=0o777)
std::string string
Definition: nybbler.cc:12
static ParameterSet make(intermediate_table const &tbl)
Definition: ParameterSet.cc:68
void ConvertChargeVectorsToPixelArray(std::vector< float > &v0pe, std::vector< float > &v1pe, std::vector< float > &v2pe, std::vector< unsigned char > &pix)
void SetViewReversal(bool reverseX, bool reverseY, bool reverseZ)
Function to set any views that need reversing.
void SetLogScale(bool setLog)
Set the log scale for charge.
error
Definition: include.cc:26
STL namespace.
int errno
Contains the last error code.
Definition: structcmd.h:53
typename config_impl< T >::type Config
Definition: ModuleMacros.h:52
const double e
static int input(void)
Definition: code.cpp:15695
Utilities for producing images for the CVN.
static Config * config
Definition: config.cpp:1054
int fTDCLimit
Limit the number of TDCs in the output image.
Definition: cvnCreateDB.cc:129
std::string fOutputDir
Class containing some utility functions for all things CVN.
Definition: CVNImageUtils.h:24
unsigned int fNEvents
Limit the number of entries in the tree to consider.
Definition: cvnCreateDB.cc:125
std::vector< bool > fReverseViews
Views to reverse.
Definition: cvnCreateDB.cc:131
int fPlaneLimit
Limit the number of wires in the output image.
Definition: cvnCreateDB.cc:127
std::string fTrainingBranchObjectName
Definition: cvnCreateDB.cc:98
int main(int argc, char *argv[])
intermediate_table parse_document(std::string const &filename, cet::filepath_maker &maker)
Definition: parse.cc:720
fhicl::ParameterSet getPSet(std::string configPath)
po::variables_map getOptions(int argc, char *argv[], std::string &config, std::string &input)
auto const & get(AssnsNode< L, R, D > const &r)
Definition: AssnsNode.h:115
int bool
Definition: qglobal.h:345
std::string to_string(ModuleType const mt)
Definition: ModuleType.h:34
QTextStream & endl(QTextStream &s)
void SetPixelMapSize(unsigned int nWires, unsigned int nTDCs)
Set the input pixel map size.
std::string fTreeName
Definition: cvnCreateDB.cc:97