lartriton::TritonClient Class Reference

#include <TritonClient.h>

Classes
struct	ServerSideStats

Public Member Functions
	TritonClient (const fhicl::ParameterSet &params)
TritonInputMap &	input ()
const TritonOutputMap &	output () const
unsigned	batchSize () const
bool	verbose () const
bool	setBatchSize (unsigned bsize)
void	dispatch ()
void	reset ()

Protected Member Functions
bool	getResults (std::shared_ptr< nvidia::inferenceserver::client::InferResult > results)
void	start ()
void	evaluate ()
void	finish (bool success)
void	reportServerSideStats (const ServerSideStats &stats) const
ServerSideStats	summarizeServerStats (const inference::ModelStatistics &start_status, const inference::ModelStatistics &end_status) const
inference::ModelStatistics	getServerSideStatus () const

Protected Attributes
TritonInputMap	input_
TritonOutputMap	output_
unsigned	allowedTries_
unsigned	tries_
std::string	serverURL_
unsigned	maxBatchSize_
unsigned	batchSize_
bool	noBatch_
bool	verbose_
std::vector< nvidia::inferenceserver::client::InferInput * >	inputsTriton_
std::vector< const nvidia::inferenceserver::client::InferRequestedOutput * >	outputsTriton_
std::unique_ptr< nvidia::inferenceserver::client::InferenceServerGrpcClient >	client_
nvidia::inferenceserver::client::InferOptions	options_

Detailed Description

Definition at line 18 of file TritonClient.h.

Constructor & Destructor Documentation

lartriton::TritonClient::TritonClient

(

const fhicl::ParameterSet &

params

)

Definition at line 27 of file TritonClient.cc.

    : allowedTries_(params.get<unsigned>("allowedTries", 0)),
      serverURL_(params.get<std::string>("serverURL")),
      verbose_(params.get<bool>("verbose")),
      options_(params.get<std::string>("modelName")) {
  //get appropriate server for this model
  if (verbose_)
    MF_LOG_INFO("TritonClient") << "Using server: " << serverURL_;

  //connect to the server
  //TODO: add SSL options
  triton_utils::throwIfError(nic::InferenceServerGrpcClient::Create(&client_, serverURL_, false),
                             "TritonClient(): unable to create inference context");

  //set options
  options_.model_version_ = params.get<std::string>("modelVersion");
  //convert seconds to microseconds
  options_.client_timeout_ = params.get<unsigned>("timeout") * 1e6;

  //config needed for batch size
  inference::ModelConfigResponse modelConfigResponse;
  triton_utils::throwIfError(client_->ModelConfig(&modelConfigResponse, options_.model_name_, options_.model_version_),
                             "TritonClient(): unable to get model config");
  inference::ModelConfig modelConfig(modelConfigResponse.config());

  //check batch size limitations (after i/o setup)
  //triton uses max batch size = 0 to denote a model that does not support batching
  //but for models that do support batching, a given event may set batch size 0 to indicate no valid input is present
  //so set the local max to 1 and keep track of "no batch" case
  maxBatchSize_ = modelConfig.max_batch_size();
  noBatch_ = maxBatchSize_ == 0;
  maxBatchSize_ = std::max(1u, maxBatchSize_);

  //get model info
  inference::ModelMetadataResponse modelMetadata;
  triton_utils::throwIfError(client_->ModelMetadata(&modelMetadata, options_.model_name_, options_.model_version_),
                             "TritonClient(): unable to get model metadata");

  //get input and output (which know their sizes)
  const auto& nicInputs = modelMetadata.inputs();
  const auto& nicOutputs = modelMetadata.outputs();

  //report all model errors at once
  std::ostringstream msg;
  std::string msg_str;

  //currently no use case is foreseen for a model with zero inputs or outputs
  if (nicInputs.empty())
    msg << "Model on server appears malformed (zero inputs)\n";

  if (nicOutputs.empty())
    msg << "Model on server appears malformed (zero outputs)\n";

  //stop if errors
  msg_str = msg.str();
  if (!msg_str.empty())
    throw cet::exception("ModelErrors") << msg_str;

  //setup input map
  std::ostringstream io_msg;
  if (verbose_)
    io_msg << "Model inputs: "
           << "\n";
  inputsTriton_.reserve(nicInputs.size());
  for (const auto& nicInput : nicInputs) {
    const auto& iname = nicInput.name();
    auto [curr_itr, success] = input_.try_emplace(iname, iname, nicInput, noBatch_);
    auto& curr_input = curr_itr->second;
    inputsTriton_.push_back(curr_input.data());
    if (verbose_) {
      io_msg << "  " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize()
             << " b) : " << triton_utils::printColl(curr_input.shape()) << "\n";
    }
  }

  //allow selecting only some outputs from server
  const auto& v_outputs = params.get<std::vector<std::string>>("outputs");
  std::unordered_set<std::string> s_outputs(v_outputs.begin(), v_outputs.end());

  //setup output map
  if (verbose_)
    io_msg << "Model outputs: "
           << "\n";
  outputsTriton_.reserve(nicOutputs.size());
  for (const auto& nicOutput : nicOutputs) {
    const auto& oname = nicOutput.name();
    if (!s_outputs.empty() and s_outputs.find(oname) == s_outputs.end())
      continue;
    auto [curr_itr, success] = output_.try_emplace(oname, oname, nicOutput, noBatch_);
    auto& curr_output = curr_itr->second;
    outputsTriton_.push_back(curr_output.data());
    if (verbose_) {
      io_msg << "  " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize()
             << " b) : " << triton_utils::printColl(curr_output.shape()) << "\n";
    }
    if (!s_outputs.empty())
      s_outputs.erase(oname);
  }

  //check if any requested outputs were not available
  if (!s_outputs.empty())
    throw cet::exception("MissingOutput")
        << "Some requested outputs were not available on the server: " << triton_utils::printColl(s_outputs);

  //propagate batch size to inputs and outputs
  setBatchSize(1);

  //print model info
  if (verbose_) {
    std::ostringstream model_msg;
    model_msg << "Model name: " << options_.model_name_ << "\n"
              << "Model version: " << options_.model_version_ << "\n"
              << "Model max batch size: " << (noBatch_ ? 0 : maxBatchSize_) << "\n";
    MF_LOG_INFO("TritonClient") << model_msg.str() << io_msg.str();
  }
}

Member Function Documentation

unsigned lartriton::TritonClient::batchSize ( ) const

inline

Definition at line 37 of file TritonClient.h.

{ return batchSize_; }

void lartriton::TritonClient::dispatch ( )

inline

Definition at line 42 of file TritonClient.h.

                  {
    start();
    evaluate();
  }

void lartriton::TritonClient::evaluate ( )

protected

Definition at line 193 of file TritonClient.cc.

                            {
  //in case there is nothing to process
  if (batchSize_ == 0) {
    finish(true);
    return;
  }

  // Get the status of the server prior to the request being made.
  const auto& start_status = getServerSideStatus();

  //blocking call
  auto t1 = std::chrono::steady_clock::now();
  nic::InferResult* results;
  bool status = triton_utils::warnIfError(client_->Infer(&results, options_, inputsTriton_, outputsTriton_),
                                          "evaluate(): unable to run and/or get result");
  if (!status) {
    finish(false);
    return;
  }

  auto t2 = std::chrono::steady_clock::now();
  MF_LOG_DEBUG("TritonClient") << "Remote time: "
                               << std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1).count();

  const auto& end_status = getServerSideStatus();

  if (verbose()) {
    const auto& stats = summarizeServerStats(start_status, end_status);
    reportServerSideStats(stats);
  }

  std::shared_ptr<nic::InferResult> results_ptr(results);
  status = getResults(results_ptr);

  //status = getResults(std::make_shared<nvidia::inferenceserver::client::InferResult> (results));
  //status = getResults(std::make_shared<nic::InferResult> (results));

  finish(status);
}

void lartriton::TritonClient::finish ( bool  success )

protected

Definition at line 233 of file TritonClient.cc.

                                      {
  //retries are only allowed if no exception was raised
  if (!success) {
    ++tries_;
    //if max retries has not been exceeded, call evaluate again
    if (tries_ < allowedTries_) {
      evaluate();
      //avoid calling doneWaiting() twice
      return;
    }
    //prepare an exception if exceeded
    throw cet::exception("TritonClient")
          << "call failed after max " << tries_ << " tries" << std::endl;
  }
}

bool lartriton::TritonClient::getResults ( std::shared_ptr< nvidia::inferenceserver::client::InferResult >  results )

protected

Definition at line 170 of file TritonClient.cc.

                                                                   {
  for (auto& [oname, output] : output_) {
    //set shape here before output becomes const
    if (output.variableDims()) {
      std::vector<int64_t> tmp_shape;
      bool status = triton_utils::warnIfError(results->Shape(oname, &tmp_shape),
                                              "getResults(): unable to get output shape for " + oname);
      if (!status)
        return status;
      output.setShape(tmp_shape, false);
    }
    //extend lifetime
    output.setResult(results);
  }

  return true;
}

inference::ModelStatistics lartriton::TritonClient::getServerSideStatus ( ) const

protected

Definition at line 306 of file TritonClient.cc.

                                                                 {
  if (verbose_) {
    inference::ModelStatisticsResponse resp;
    bool success = triton_utils::warnIfError(
        client_->ModelInferenceStatistics(&resp, options_.model_name_, options_.model_version_),
        "getServerSideStatus(): unable to get model statistics");
    if (success)
      return *(resp.model_stats().begin());
  }
  return inference::ModelStatistics{};
}

TritonInputMap& lartriton::TritonClient::input ( )

inline

Definition at line 35 of file TritonClient.h.

{ return input_; }

const TritonOutputMap& lartriton::TritonClient::output ( ) const

inline

Definition at line 36 of file TritonClient.h.

{ return output_; }

void lartriton::TritonClient::reportServerSideStats ( const ServerSideStats &  stats ) const

protected

Definition at line 249 of file TritonClient.cc.

                                                                                       {
  std::ostringstream msg;

  // https://github.com/triton-inference-server/server/blob/v2.3.0/src/clients/c++/perf_client/inference_profiler.cc
  const uint64_t count = stats.success_count_;
  msg << "  Inference count: " << stats.inference_count_ << "\n";
  msg << "  Execution count: " << stats.execution_count_ << "\n";
  msg << "  Successful request count: " << count << "\n";

  if (count > 0) {
    auto get_avg_us = [count](uint64_t tval) {
      constexpr uint64_t us_to_ns = 1000;
      return tval / us_to_ns / count;
    };

    const uint64_t cumm_avg_us = get_avg_us(stats.cumm_time_ns_);
    const uint64_t queue_avg_us = get_avg_us(stats.queue_time_ns_);
    const uint64_t compute_input_avg_us = get_avg_us(stats.compute_input_time_ns_);
    const uint64_t compute_infer_avg_us = get_avg_us(stats.compute_infer_time_ns_);
    const uint64_t compute_output_avg_us = get_avg_us(stats.compute_output_time_ns_);
    const uint64_t compute_avg_us = compute_input_avg_us + compute_infer_avg_us + compute_output_avg_us;
    const uint64_t overhead =
        (cumm_avg_us > queue_avg_us + compute_avg_us) ? (cumm_avg_us - queue_avg_us - compute_avg_us) : 0;

    msg << "  Avg request latency: " << cumm_avg_us << " usec"
        << "\n"
        << "  (overhead " << overhead << " usec + "
        << "queue " << queue_avg_us << " usec + "
        << "compute input " << compute_input_avg_us << " usec + "
        << "compute infer " << compute_infer_avg_us << " usec + "
        << "compute output " << compute_output_avg_us << " usec)" << std::endl;
  }

  MF_LOG_DEBUG("TritonClient") << msg.str();
}

void lartriton::TritonClient::reset

(

)

Definition at line 161 of file TritonClient.cc.

                         {
  for (auto& element : input_) {
    element.second.reset();
  }
  for (auto& element : output_) {
    element.second.reset();
  }
}

bool lartriton::TritonClient::setBatchSize

(

unsigned

bsize

)

Definition at line 144 of file TritonClient.cc.

                                              {
  if (bsize > maxBatchSize_) {
    MF_LOG_WARNING("TritonClient") << "Requested batch size " << bsize << " exceeds server-specified max batch size "
                                   << maxBatchSize_ << ". Batch size will remain as" << batchSize_;
    return false;
  }
  batchSize_ = bsize;
  //set for input and output
  for (auto& element : input_) {
    element.second.setBatchSize(bsize);
  }
  for (auto& element : output_) {
    element.second.setBatchSize(bsize);
  }
  return true;
}

void lartriton::TritonClient::start ( void  )

protected

Definition at line 188 of file TritonClient.cc.

                         {
  tries_ = 0;
}

TritonClient::ServerSideStats lartriton::TritonClient::summarizeServerStats ( const inference::ModelStatistics &  start_status, const inference::ModelStatistics &  end_status  ) const

protected

Definition at line 285 of file TritonClient.cc.

                                                                                                                   {
  TritonClient::ServerSideStats server_stats;

  server_stats.inference_count_ = end_status.inference_count() - start_status.inference_count();
  server_stats.execution_count_ = end_status.execution_count() - start_status.execution_count();
  server_stats.success_count_ =
      end_status.inference_stats().success().count() - start_status.inference_stats().success().count();
  server_stats.cumm_time_ns_ =
      end_status.inference_stats().success().ns() - start_status.inference_stats().success().ns();
  server_stats.queue_time_ns_ = end_status.inference_stats().queue().ns() - start_status.inference_stats().queue().ns();
  server_stats.compute_input_time_ns_ =
      end_status.inference_stats().compute_input().ns() - start_status.inference_stats().compute_input().ns();
  server_stats.compute_infer_time_ns_ =
      end_status.inference_stats().compute_infer().ns() - start_status.inference_stats().compute_infer().ns();
  server_stats.compute_output_time_ns_ =
      end_status.inference_stats().compute_output().ns() - start_status.inference_stats().compute_output().ns();

  return server_stats;
}

bool lartriton::TritonClient::verbose ( ) const

inline

Definition at line 38 of file TritonClient.h.

{ return verbose_; }

Member Data Documentation

unsigned lartriton::TritonClient::allowedTries_

protected

Definition at line 67 of file TritonClient.h.

unsigned lartriton::TritonClient::batchSize_

protected

Definition at line 70 of file TritonClient.h.

std::unique_ptr<nvidia::inferenceserver::client::InferenceServerGrpcClient> lartriton::TritonClient::client_

protected

Definition at line 78 of file TritonClient.h.

TritonInputMap lartriton::TritonClient::input_

protected

Definition at line 65 of file TritonClient.h.

std::vector<nvidia::inferenceserver::client::InferInput*> lartriton::TritonClient::inputsTriton_

protected

Definition at line 75 of file TritonClient.h.

unsigned lartriton::TritonClient::maxBatchSize_

protected

Definition at line 69 of file TritonClient.h.

bool lartriton::TritonClient::noBatch_

protected

Definition at line 71 of file TritonClient.h.

nvidia::inferenceserver::client::InferOptions lartriton::TritonClient::options_

protected

Definition at line 80 of file TritonClient.h.

TritonOutputMap lartriton::TritonClient::output_

protected

Definition at line 66 of file TritonClient.h.

std::vector<const nvidia::inferenceserver::client::InferRequestedOutput*> lartriton::TritonClient::outputsTriton_

protected

Definition at line 76 of file TritonClient.h.

std::string lartriton::TritonClient::serverURL_

protected

Definition at line 68 of file TritonClient.h.

unsigned lartriton::TritonClient::tries_

protected

Definition at line 67 of file TritonClient.h.

bool lartriton::TritonClient::verbose_

protected

Definition at line 72 of file TritonClient.h.

---

The documentation for this class was generated from the following files:

• larrecodnn/larrecodnn/ImagePatternAlgs/NuSonic/Triton/TritonClient.h
• larrecodnn/larrecodnn/ImagePatternAlgs/NuSonic/Triton/TritonClient.cc