#include <InputFileCatalog.h>

Inheritance diagram for art::InputFileCatalog:

Classes
struct	Config

Public Member Functions
	InputFileCatalog (fhicl::TableFragment< Config > const &config)

virtual	~InputFileCatalog ()=default

std::size_t	size () const

FileCatalogItem const &	currentFile () const

size_t	currentIndex () const

bool	getNextFile (int attempts=5)

bool	hasNextFile (int attempts=5)

void	rewind ()

void	rewindTo (size_t index)

bool	isSearchable ()

bool	empty () const

void	finish ()

std::vector< std::string > const &	fileSources () const

Public Member Functions inherited from art::FileCatalog
virtual	~FileCatalog ()=default

void	commitCatalog ()

std::string &	url ()

std::string const &	url () const

void	setActive ()

bool	active () const

Static Public Attributes
static constexpr size_t	indexEnd {std::numeric_limits<size_t>::max()}

Private Member Functions
bool	retrieveNextFile (FileCatalogItem &item, int attempts, bool transferOnly=false)

FileCatalogStatus	retrieveNextFileFromCacheOrService (FileCatalogItem &item)

FileCatalogStatus	transferNextFile (FileCatalogItem &item)

Private Attributes
std::vector< std::string >	fileSources_

std::vector< FileCatalogItem >	fileCatalogItems_ {{}}

FileCatalogItem	nextItem_ {}

size_t	fileIdx_ {indexEnd}

size_t	maxIdx_ {}

bool	searchable_ {false}

bool	nextFileProbed_ {false}

bool	hasNextFile_ {false}

ServiceHandle< CatalogInterface >	ci_

ServiceHandle< FileTransfer >	ft_

Additional Inherited Members
Static Public Member Functions inherited from art::FileCatalog
static bool	isPhysical (std::string const &name)

Detailed Description

Definition at line 30 of file InputFileCatalog.h.

Constructor & Destructor Documentation

art::InputFileCatalog::InputFileCatalog ( fhicl::TableFragment< Config > const & config )

explicit

Definition at line 20 of file InputFileCatalog.cc.

     : fileSources_{config().namesParameter()}
   {
 
     if (fileSources_.empty()) {
       throw art::Exception(art::errors::CatalogServiceError,
                            "InputFileCatalog::InputFileCatalog()\n")
         << "Empty '" << config().namesParameter.name()
         << "' parameter specified for input source.\n";
     }
 
     // Configure FileDelivery service
     ci_->configure(fileSources_);
     searchable_ = ci_->isSearchable();
 
     if (searchable_)
       fileCatalogItems_.resize(fileSources_.size());
   }

virtual art::InputFileCatalog::~InputFileCatalog ( )

virtualdefault

Member Function Documentation

FileCatalogItem const & art::InputFileCatalog::currentFile ( ) const

Definition at line 41 of file InputFileCatalog.cc.

   {
     if (fileIdx_ == indexEnd) {
       throw art::Exception(
         art::errors::LogicError,
         "Cannot access the current file while the file catalog is empty!");
     }
     assert(fileIdx_ <= maxIdx_);
     return fileCatalogItems_[fileIdx_];
   }

size_t art::InputFileCatalog::currentIndex ( ) const

Definition at line 53 of file InputFileCatalog.cc.

   {
     return fileIdx_;
   }

bool art::InputFileCatalog::empty ( ) const

inline

Definition at line 55 of file InputFileCatalog.h.

     {
       return fileCatalogItems_.empty();
     }

std::vector<std::string> const& art::InputFileCatalog::fileSources ( ) const

inline

Definition at line 61 of file InputFileCatalog.h.

     {
       return fileSources_;
     }

void art::InputFileCatalog::finish ( )

Definition at line 248 of file InputFileCatalog.cc.

   {
     if (fileIdx_ != indexEnd          // there is a current file
         && !currentFile().skipped()   // not skipped
         && !currentFile().consumed()) // not consumed
     {
       ci_->updateStatus(currentFile().uri(), FileDisposition::CONSUMED);
       fileCatalogItems_[fileIdx_].consume();
     }
   }

bool art::InputFileCatalog::getNextFile ( int attempts = 5 )

Definition at line 59 of file InputFileCatalog.cc.

   {
     // get next file from FileDelivery service and give it to
     // currentFile_ object returns false if theres no more file
     //
     // If hasNextFile() has been called prior to getNextFile(), it
     // does not actually go fetch the next file from FileDelivery
     // service, instead, it advances the iterator by one and make the
     // "hidden" next file current.
 
     if (nextFileProbed_ && !hasNextFile_)
       return false;
 
     if ((nextFileProbed_ && hasNextFile_) ||
         retrieveNextFile(nextItem_, attempts)) {
       nextFileProbed_ = false;
       fileIdx_ =
         (fileIdx_ == indexEnd) ? 0 : (searchable_ ? (fileIdx_ + 1) : 0);
       if (fileIdx_ > maxIdx_)
         maxIdx_ = fileIdx_;
       fileCatalogItems_[fileIdx_] = nextItem_;
       return true;
     }
 
     return false;
   }

bool art::InputFileCatalog::hasNextFile ( int attempts = 5 )

Definition at line 87 of file InputFileCatalog.cc.

   {
     // A probe. It tries (and actually does) retrieve the next file
     // from the FileDelivery service. But does not advance the current
     // file pointer
     if (nextFileProbed_)
       return hasNextFile_;
 
     hasNextFile_ = retrieveNextFile(nextItem_, attempts);
     nextFileProbed_ = true;
 
     return hasNextFile_;
   }

bool art::InputFileCatalog::isSearchable ( )

inline

Definition at line 50 of file InputFileCatalog.h.

     {
       return searchable_;
     }

bool art::InputFileCatalog::retrieveNextFile	(	FileCatalogItem &	item,
		int	attempts,
		bool	transferOnly = `false`
	)

private

Definition at line 102 of file InputFileCatalog.cc.

   {
     // Tell the service the current opened file (if there is one) is consumed
     finish();
 
     // retrieve (deliver and transfer) next file from service
     // or, do the transfer only
     FileCatalogStatus status;
     if (transferOnly) {
       status = transferNextFile(item);
     } else {
       status = retrieveNextFileFromCacheOrService(item);
     }
 
     switch (status) {
     case FileCatalogStatus::SUCCESS: {
       // mark the file as transferred
       ci_->updateStatus(item.uri(), FileDisposition::TRANSFERRED);
       return true;
     }
     case FileCatalogStatus::NO_MORE_FILES: {
       return false;
     }
     case FileCatalogStatus::DELIVERY_ERROR: {
       if (attempts <= 1) {
         throw art::Exception(art::errors::Configuration,
                              "InputFileCatalog::retrieveNextFile()\n")
           << "Delivery error encountered after reaching maximum number of "
              "attemtps!";
       } else {
         return retrieveNextFile(item, attempts - 1, false);
       }
     }
     case FileCatalogStatus::TRANSFER_ERROR: {
       if (attempts <= 1) {
         // if we end up with a transfer error, the method returns
         // with a true flag and empty filename. Weired enough, but
         // the next file does exist we just cannot retrieve it. Therefore
         // we notify the service that the file has been skipped
         ci_->updateStatus(item.uri(), FileDisposition::SKIPPED);
         return true;
       } else {
         return retrieveNextFile(item, attempts - 1, true);
       }
     }
     }
     assert(false);
     return false; // Unreachable, but keeps the compiler happy
   }

FileCatalogStatus art::InputFileCatalog::retrieveNextFileFromCacheOrService ( FileCatalogItem & item )

private

Definition at line 155 of file InputFileCatalog.cc.

   {
     // Try to get it from cached files
     if (fileIdx_ < maxIdx_) {
       item = fileCatalogItems_[fileIdx_ + 1];
       return FileCatalogStatus::SUCCESS;
     }
 
     // Try to get it from the service
     std::string uri;
     double wait = 0.0;
 
     // get file delivered
     int const result = ci_->getNextFileURI(uri, wait);
 
     if (result == FileDeliveryStatus::NO_MORE_FILES)
       return FileCatalogStatus::NO_MORE_FILES;
 
     if (result != FileDeliveryStatus::SUCCESS)
       return FileCatalogStatus::DELIVERY_ERROR;
 
     item = FileCatalogItem("", "", uri);
 
     // get file transfered
     return transferNextFile(item);
   }

void art::InputFileCatalog::rewind ( )

Definition at line 216 of file InputFileCatalog.cc.

   {
     if (!searchable_) {
       throw art::Exception(art::errors::LogicError,
                            "InputFileCatalog::rewind()\n")
         << "A non-searchable catalog is not allowed to rewind!";
     }
     fileIdx_ = 0;
   }

void art::InputFileCatalog::rewindTo ( size_t index )

Definition at line 227 of file InputFileCatalog.cc.

   {
     // rewind to a previous file location in the catalog service is
     // not rewinded. only usable when
     // FileDeliveryService::areFilesPersistent() is true
     if (!searchable_) {
       throw art::Exception(art::errors::LogicError,
                            "InputFileCatalog::rewindTo()\n")
         << "A non-searchable catalog is not allowed to rewind!";
     }
 
     if (index > maxIdx_) {
       throw art::Exception(art::errors::InvalidNumber,
                            "InputFileCatalog::rewindTo()\n")
         << "Index " << index << " is out of range!";
     }
 
     fileIdx_ = index;
   }

std::size_t art::InputFileCatalog::size ( ) const

inline

Definition at line 39 of file InputFileCatalog.h.

     {
       return fileCatalogItems_.size();
     }

FileCatalogStatus art::InputFileCatalog::transferNextFile ( FileCatalogItem & item )

private

Definition at line 183 of file InputFileCatalog.cc.

   {
     std::string pfn;
     int const result = ft_->translateToLocalFilename(item.uri(), pfn);
 
     if (result != FileTransferStatus::SUCCESS) {
       item.fileName("");
       item.logicalFileName("");
       item.skip();
       return FileCatalogStatus::TRANSFER_ERROR;
     }
 
     // successfully retrieved the file
     std::string lfn = pfn;
     boost::trim(pfn);
 
     if (pfn.empty()) {
       throw art::Exception(
         art::errors::Configuration,
         "InputFileCatalog::retrieveNextFileFromCacheService()\n")
         << "An empty string specified in parameter for input source.\n";
     }
 
     if (isPhysical(pfn)) {
       lfn.clear();
     }
 
     item.fileName(pfn);
     item.logicalFileName(lfn);
     return FileCatalogStatus::SUCCESS;
   }

Member Data Documentation

ServiceHandle<CatalogInterface> art::InputFileCatalog::ci_

private

Definition at line 84 of file InputFileCatalog.h.

std::vector<FileCatalogItem> art::InputFileCatalog::fileCatalogItems_ {{}}

private

Definition at line 76 of file InputFileCatalog.h.

size_t art::InputFileCatalog::fileIdx_ {indexEnd}

private

Definition at line 78 of file InputFileCatalog.h.

std::vector<std::string> art::InputFileCatalog::fileSources_

private

Definition at line 75 of file InputFileCatalog.h.

ServiceHandle<FileTransfer> art::InputFileCatalog::ft_

private

Definition at line 85 of file InputFileCatalog.h.

bool art::InputFileCatalog::hasNextFile_ {false}

private

Definition at line 82 of file InputFileCatalog.h.

constexpr size_t art::InputFileCatalog::indexEnd {std::numeric_limits<size_t>::max()}

static

Definition at line 66 of file InputFileCatalog.h.

size_t art::InputFileCatalog::maxIdx_ {}

private

Definition at line 79 of file InputFileCatalog.h.

bool art::InputFileCatalog::nextFileProbed_ {false}

private

Definition at line 81 of file InputFileCatalog.h.

FileCatalogItem art::InputFileCatalog::nextItem_ {}

private

Definition at line 77 of file InputFileCatalog.h.

bool art::InputFileCatalog::searchable_ {false}

private

Definition at line 80 of file InputFileCatalog.h.

The documentation for this class was generated from the following files:

art/art/Framework/IO/Catalog/InputFileCatalog.h
art/art/Framework/IO/Catalog/InputFileCatalog.cc

Classes

Public Member Functions

Static Public Attributes

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation