dune::FelixCompressor Class Reference

#include <FelixCompress.hh>

Public Member Functions
	FelixCompressor (const uint8_t *data, const size_t num_frames=10000)
	FelixCompressor (const dune::FelixFragment &frag)
void	store_metadata (std::vector< char > &out)
void	header_reduce (std::vector< char > &out)
void	generate_Huff_tree (std::vector< char > &out)
void	ADC_compress (std::vector< char > &out)
void	compress_copy (std::vector< char > &out)

Private Member Functions
FelixFrame const *	frame_ (const size_t &frame_num=0) const

Private Attributes
const void *	input
const size_t	input_length
const size_t	num_frames = input_length / sizeof(FelixFrame)
HuffTree	hufftree

Detailed Description

Definition at line 200 of file FelixCompress.hh.

Constructor & Destructor Documentation

dune::FelixCompressor::FelixCompressor ( const uint8_t *  data, const size_t  num_frames = 10000  )

inline

Definition at line 216 of file FelixCompress.hh.

      : input(data), input_length(num_frames * sizeof(dune::FelixFrame)) {}

dune::FelixCompressor::FelixCompressor ( const dune::FelixFragment &  frag )

inline

Definition at line 218 of file FelixCompress.hh.

      : input(frag.dataBeginBytes()), input_length(frag.dataSizeBytes()) {}

Member Function Documentation

void dune::FelixCompressor::ADC_compress ( std::vector< char > &  out )

inline

Definition at line 371 of file FelixCompress.hh.

                                          {
    // Resize the output buffer appropriately for the worst case.
    const size_t tail = out.size();
    out.resize(tail + sizeof(adc_t) * num_frames * 256);
    // Record the encoded ADC values into the buffer.
    unsigned rec_bits = 0;
    const char* dest = &out[0] + tail;

    for(unsigned j = 0; j < 256; ++j) {
      adc_t curr_val = frame_(0)->channel(j);
      *(unsigned long*)(dest + (rec_bits/8)) |= hufftree(curr_val)->huffcode << (rec_bits%8);
      rec_bits += hufftree(curr_val)->hufflength;

      for (unsigned i = 1; i < num_frames; ++i) {
#ifdef PREV
        adc_t curr_val = frame_(i)->channel(j) - frame_(i - 1)->channel(j);
#else
        adc_t curr_val = frame_(i)->channel(j);
#endif

        // No code is longer than 64 bits.
        *(uint64_t*)(dest + (rec_bits/8)) |= hufftree(curr_val)->huffcode << (rec_bits%8);
        rec_bits += hufftree(curr_val)->hufflength;
      }
    }
    // Resize output buffer to actual data length.
    out.resize(tail + rec_bits / 8 + 1);
  }

void dune::FelixCompressor::compress_copy ( std::vector< char > &  out )

inline

Definition at line 401 of file FelixCompress.hh.

                                           {
    // auto comp_begin = std::chrono::high_resolution_clock::now();
    store_metadata(out);
    // auto meta_end = std::chrono::high_resolution_clock::now();
    // std::cout << "Meta time taken: "
    //           << std::chrono::duration_cast<std::chrono::microseconds>(
    //                  meta_end - comp_begin)
    //                  .count()
    //           << " us.\n";
    header_reduce(out);
    // auto header_end = std::chrono::high_resolution_clock::now();
    // std::cout << "Header time taken: "
    //           << std::chrono::duration_cast<std::chrono::microseconds>(
    //                  header_end - meta_end)
    //                  .count()
    //           << " us.\n";
    generate_Huff_tree(out);
    // auto huff_end = std::chrono::high_resolution_clock::now();
    // std::cout << "Huffman time taken: "
    //           << std::chrono::duration_cast<std::chrono::microseconds>(
    //                  huff_end - header_end)
    //                  .count()
    //           << " us.\n";
    ADC_compress(out);
    // auto adc_end = std::chrono::high_resolution_clock::now();
    // std::cout << "ADC time taken: "
    //           << std::chrono::duration_cast<std::chrono::microseconds>(
    //                  adc_end - huff_end)
    //                  .count()
    //           << " us.\n";
  }

FelixFrame const* dune::FelixCompressor::frame_ ( const size_t &  frame_num = 0 ) const

inlineprivate

Definition at line 208 of file FelixCompress.hh.

                                                              {
    return static_cast<FelixFrame const*>(input) + frame_num;
  }

void dune::FelixCompressor::generate_Huff_tree ( std::vector< char > &  out )

inline

Definition at line 322 of file FelixCompress.hh.

                                                {
    // Build a frequency table.
    std::unordered_map<adc_t, uint32_t> freq_table;
    for (unsigned vi = 0; vi < frame_()->num_ch_per_frame; ++vi) {
      freq_table[frame_(0)->channel(vi)]++;
      for (unsigned fri = 1; fri < num_frames; ++fri) {
#ifdef PREV
        adc_t curr_val = frame_(fri)->channel(vi) - frame_(fri - 1)->channel(vi);
#else
        adc_t curr_val = frame_(fri)->channel(vi);
#endif
        freq_table[curr_val]++;
      }
    }

    // Save the number of unique values to the metadata.
    MetaData* meta = reinterpret_cast<MetaData*>(&out[0]);
    meta->unique_values = freq_table.size();

    // Save the frequency table to the outgoing buffer so that the decompressor
    // can make a Huffman tree of its own.
    size_t tail = out.size();
    out.resize(tail + freq_table.size() * (sizeof(adc_t) + 4));
    for (auto p : freq_table) {
      memcpy(&out[tail], &p.first, sizeof(p.first));
      memcpy(&out[tail + sizeof(p.first)], &p.second, sizeof(p.second));
      tail += sizeof(p.first) + sizeof(p.second);
    }

    // Insert the frequency table into Huffman tree nodes and calculate the
    // information entropy according to Shannon.
    std::vector<HuffTree::Node> nodes;
    double entropy = 0;
    const unsigned num_vals = num_frames * frame_()->num_ch_per_frame;
    for (auto p : freq_table) {
      HuffTree::Node curr_node;
      curr_node.value = p.first;
      curr_node.frequency = p.second;
      nodes.push_back(curr_node);

      entropy += (double)p.second / num_vals * log((double)num_vals / p.second);
    }
    // std::cout << "ADC value entropy: " << entropy << " bits.\n";

    // Connect the nodes in the tree according to Huffman's method.
    hufftree.make_tree(nodes);
  }

void dune::FelixCompressor::header_reduce ( std::vector< char > &  out )

inline

Definition at line 233 of file FelixCompress.hh.

                                           {
    // Fields for storing which frames have faulty headers.
    std::vector<uint8_t> bad_headers(num_frames/8 + 1);
    // Check all error fields and increments of timestamp and CCC.
    for (unsigned i = 1; i < num_frames; ++i) {
      bool check_failed = false;

      // WIB header checks.
      check_failed |= frame_()->sof() ^ frame_(i)->sof();
      check_failed |= frame_()->version() ^ frame_(i)->version();
      check_failed |= frame_()->fiber_no() ^ frame_(i)->fiber_no();
      check_failed |= frame_()->crate_no() ^ frame_(i)->crate_no();
      check_failed |= frame_()->slot_no() ^ frame_(i)->slot_no();
      check_failed |= frame_()->mm() ^ frame_(i)->mm();
      check_failed |= frame_()->oos() ^ frame_(i)->oos();
      check_failed |= frame_()->wib_errors() ^ frame_(i)->wib_errors();
      check_failed |= frame_()->z() ^ frame_(i)->z();

      check_failed |=
          (uint64_t)(frame_()->timestamp() + 25 * i) ^ frame_(i)->timestamp();

      // COLDATA header checks.
      for (unsigned j = 0; j < 4; ++j) {
        check_failed |= frame_()->s1_error(j) ^ frame_(i)->s1_error(j);
        check_failed |= frame_()->s2_error(j) ^ frame_(i)->s2_error(j);
        check_failed |= frame_()->checksum_a(j) ^ frame_(i)->checksum_a(j);
        check_failed |= frame_()->checksum_b(j) ^ frame_(i)->checksum_b(j);
        check_failed |=
            frame_()->error_register(j) ^ frame_(i)->error_register(j);
        for (unsigned h = 0; h < 8; ++h) {
          check_failed |= frame_()->hdr(j, h) ^ frame_(i)->hdr(j, h);
        }

        check_failed |=
            (uint16_t)(frame_()->coldata_convert_count(j) + 25 * i) ^
            frame_(i)->coldata_convert_count(j);
      }

      // Set a bit if anything failed.
      if (check_failed) {
        bad_headers[i/8] |= 1 << (i%8);

        // std::cout << "Check failed at the " << i << "th frame.\n";

        // std::cout << "First frame:\n";
        // frame_(i - 1)->print();
        // std::cout << "Second frame:\n";
        // frame_(i)->print();

        // exit(1);
      }
    }  // Loop over all frames.

    // Record faulty frame field.
    size_t tail = out.size();
    out.resize(tail + num_frames/8+1);
    memcpy(&out[tail], &bad_headers[0], bad_headers.size());

    // Record first headers.
    tail = out.size();
    out.resize(tail + sizeof(WIBHeader) + 4*sizeof(ColdataHeader));

    memcpy(&out[tail], frame_()->wib_header(), sizeof(WIBHeader));
    memcpy(&out[tail + sizeof(WIBHeader)], frame_()->coldata_header(0),
           sizeof(ColdataHeader));
    memcpy(&out[tail + sizeof(WIBHeader) + sizeof(ColdataHeader)], frame_()->coldata_header(1),
           sizeof(ColdataHeader));
    memcpy(&out[tail + sizeof(WIBHeader) + 2*sizeof(ColdataHeader)], frame_()->coldata_header(2),
           sizeof(ColdataHeader));
    memcpy(&out[tail + sizeof(WIBHeader) + 3*sizeof(ColdataHeader)], frame_()->coldata_header(3),
           sizeof(ColdataHeader));

    // Loop over frames again to record faulty headers.
    for(unsigned i = 1; i < num_frames; ++i) {
      bool check_failed = (bad_headers[i / 8] >> (7 - (i % 8))) & 1;
      if(!check_failed) { continue; }

      tail = out.size();
      out.resize(tail + sizeof(WIBHeader) + 4*sizeof(ColdataHeader));

      memcpy(&out[tail], frame_(i)->wib_header(), sizeof(WIBHeader));
      for(unsigned j = 0; j < 4; ++j) {
        memcpy(&out[tail + sizeof(WIBHeader)], frame_(i)->coldata_header(j),
              sizeof(ColdataHeader));
      }
    }
  }

void dune::FelixCompressor::store_metadata ( std::vector< char > &  out )

inline

Definition at line 222 of file FelixCompress.hh.

                                            {
    // Clear the vector as a first action to make sure there is a clean slate to
    // work with.
    out.clear();

    MetaData meta = {1, 0, (uint32_t)num_frames};
    out.resize(sizeof(meta));
    memcpy(&out[0], &meta, sizeof(meta));
  }

Member Data Documentation

HuffTree dune::FelixCompressor::hufftree

private

Definition at line 213 of file FelixCompress.hh.

const void* dune::FelixCompressor::input

private

Definition at line 203 of file FelixCompress.hh.

const size_t dune::FelixCompressor::input_length

private

Definition at line 204 of file FelixCompress.hh.

const size_t dune::FelixCompressor::num_frames = input_length / sizeof(FelixFrame)

private

Definition at line 205 of file FelixCompress.hh.

---

The documentation for this class was generated from the following file:

• dune-raw-data/dune-raw-data/Overlays/FelixCompress.hh