c2numpy.h

Go to the documentation of this file.

// Copyright 2016 Jim Pivarski
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef C2NUMPY
#define C2NUMPY

#include <inttypes.h>
#include <stdarg.h>
#include <string.h>

#include <sstream>
#include <string>
#include <vector>

const char* C2NUMPY_VERSION = "1.2";

// http://docs.scipy.org/doc/numpy/user/basics.types.html
typedef enum {
    C2NUMPY_BOOL,        // Boolean (True or False) stored as a byte
    C2NUMPY_INT,         // Default integer type (same as C long; normally either int64 or int32)
    C2NUMPY_INTC,        // Identical to C int (normally int32 or int64)
    C2NUMPY_INTP,        // Integer used for indexing (same as C ssize_t; normally either int32 or int64)
    C2NUMPY_INT8,        // Byte (-128 to 127)
    C2NUMPY_INT16,       // Integer (-32768 to 32767)
    C2NUMPY_INT32,       // Integer (-2147483648 to 2147483647)
    C2NUMPY_INT64,       // Integer (-9223372036854775808 to 9223372036854775807)
    C2NUMPY_UINT8,       // Unsigned integer (0 to 255)
    C2NUMPY_UINT16,      // Unsigned integer (0 to 65535)
    C2NUMPY_UINT32,      // Unsigned integer (0 to 4294967295)
    C2NUMPY_UINT64,      // Unsigned integer (0 to 18446744073709551615)
    C2NUMPY_FLOAT,       // Shorthand for float64.
    C2NUMPY_FLOAT16,     // Half precision float: sign bit, 5 bits exponent, 10 bits mantissa
    C2NUMPY_FLOAT32,     // Single precision float: sign bit, 8 bits exponent, 23 bits mantissa
    C2NUMPY_FLOAT64,     // Double precision float: sign bit, 11 bits exponent, 52 bits mantissa
    C2NUMPY_COMPLEX,     // Shorthand for complex128.
    C2NUMPY_COMPLEX64,   // Complex number, represented by two 32-bit floats (real and imaginary components)
    C2NUMPY_COMPLEX128,  // Complex number, represented by two 64-bit floats (real and imaginary components)

    C2NUMPY_STRING       = 100,  // strings are C2NUMPY_STRING + their fixed size (up to 155)
    C2NUMPY_END          = 255   // ensure that c2numpy_type is at least a byte
} c2numpy_type;

// a Numpy writer object
typedef struct {
    FILE *file;                   // output file handle
    std::string outputFilePrefix;       // output file name, not including the rotating number and .npy
    int64_t sizeSeekPosition;     // (internal) keep track of number of rows to modify before closing
    int64_t sizeSeekSize;         // (internal)

    int32_t numColumns;           // number of columns in the record array
    std::vector<std::string> columnNames;           // column names
    std::vector<c2numpy_type> columnTypes;    // column types

    int32_t numRowsPerFile;       // maximum number of rows per file
    int32_t currentColumn;        // current column number
    int32_t currentRowInFile;     // current row number in the current file
    int32_t currentFileNumber;    // current file number
} c2numpy_writer;

const char *c2numpy_descr(c2numpy_type type) {
    // FIXME: all of the "<" signs should be system-dependent (they mean little endian)
    static const char *c2numpy_bool = "|b1";
    static const char *c2numpy_int = "<i8";
    static const char *c2numpy_intc = "<i4";   // FIXME: should be system-dependent
    static const char *c2numpy_intp = "<i8";   // FIXME: should be system-dependent
    static const char *c2numpy_int8 = "|i1";
    static const char *c2numpy_int16 = "<i2";
    static const char *c2numpy_int32 = "<i4";
    static const char *c2numpy_int64 = "<i8";
    static const char *c2numpy_uint8 = "|u1";
    static const char *c2numpy_uint16 = "<u2";
    static const char *c2numpy_uint32 = "<u4";
    static const char *c2numpy_uint64 = "<u8";
    static const char *c2numpy_float = "<f8";
    static const char *c2numpy_float16 = "<f2";
    static const char *c2numpy_float32 = "<f4";
    static const char *c2numpy_float64 = "<f8";
    static const char *c2numpy_complex = "<c16";
    static const char *c2numpy_complex64 = "<c8";
    static const char *c2numpy_complex128 = "<c16";

    static const char *c2numpy_str[155] = {"|S0", "|S1", "|S2", "|S3", "|S4", "|S5", "|S6", "|S7", "|S8", "|S9", "|S10", "|S11", "|S12", "|S13", "|S14", "|S15", "|S16", "|S17", "|S18", "|S19", "|S20", "|S21", "|S22", "|S23", "|S24", "|S25", "|S26", "|S27", "|S28", "|S29", "|S30", "|S31", "|S32", "|S33", "|S34", "|S35", "|S36", "|S37", "|S38", "|S39", "|S40", "|S41", "|S42", "|S43", "|S44", "|S45", "|S46", "|S47", "|S48", "|S49", "|S50", "|S51", "|S52", "|S53", "|S54", "|S55", "|S56", "|S57", "|S58", "|S59", "|S60", "|S61", "|S62", "|S63", "|S64", "|S65", "|S66", "|S67", "|S68", "|S69", "|S70", "|S71", "|S72", "|S73", "|S74", "|S75", "|S76", "|S77", "|S78", "|S79", "|S80", "|S81", "|S82", "|S83", "|S84", "|S85", "|S86", "|S87", "|S88", "|S89", "|S90", "|S91", "|S92", "|S93", "|S94", "|S95", "|S96", "|S97", "|S98", "|S99", "|S100", "|S101", "|S102", "|S103", "|S104", "|S105", "|S106", "|S107", "|S108", "|S109", "|S110", "|S111", "|S112", "|S113", "|S114", "|S115", "|S116", "|S117", "|S118", "|S119", "|S120", "|S121", "|S122", "|S123", "|S124", "|S125", "|S126", "|S127", "|S128", "|S129", "|S130", "|S131", "|S132", "|S133", "|S134", "|S135", "|S136", "|S137", "|S138", "|S139", "|S140", "|S141", "|S142", "|S143", "|S144", "|S145", "|S146", "|S147", "|S148", "|S149", "|S150", "|S151", "|S152", "|S153", "|S154"};

    switch (type) {
      case C2NUMPY_BOOL:
          return c2numpy_bool;
      case C2NUMPY_INT:
          return c2numpy_int;
      case C2NUMPY_INTC:
          return c2numpy_intc;
      case C2NUMPY_INTP:
          return c2numpy_intp;
      case C2NUMPY_INT8:
          return c2numpy_int8;
      case C2NUMPY_INT16:
          return c2numpy_int16;
      case C2NUMPY_INT32:
          return c2numpy_int32;
      case C2NUMPY_INT64:
          return c2numpy_int64;
      case C2NUMPY_UINT8:
          return c2numpy_uint8;
      case C2NUMPY_UINT16:
          return c2numpy_uint16;
      case C2NUMPY_UINT32:
          return c2numpy_uint32;
      case C2NUMPY_UINT64:
          return c2numpy_uint64;
      case C2NUMPY_FLOAT:
          return c2numpy_float;
      case C2NUMPY_FLOAT16:
          return c2numpy_float16;
      case C2NUMPY_FLOAT32:
          return c2numpy_float32;
      case C2NUMPY_FLOAT64:
          return c2numpy_float64;
      case C2NUMPY_COMPLEX:
          return c2numpy_complex;
      case C2NUMPY_COMPLEX64:
          return c2numpy_complex64;
      case C2NUMPY_COMPLEX128:
          return c2numpy_complex128;
      default:
          if (0 < type - C2NUMPY_STRING  &&  type - C2NUMPY_STRING < 155)
              return c2numpy_str[type - C2NUMPY_STRING];
    }

    return NULL;
}

int c2numpy_init(c2numpy_writer *writer, const std::string outputFilePrefix, int32_t numRowsPerFile) {
    writer->file = NULL;
    writer->outputFilePrefix = outputFilePrefix;
    writer->sizeSeekPosition = 0;
    writer->sizeSeekSize = 0;

    writer->numColumns = 0;

    writer->numRowsPerFile = numRowsPerFile;
    writer->currentColumn = 0;
    writer->currentRowInFile = 0;
    writer->currentFileNumber = 0;

    return 0;
}

int c2numpy_addcolumn(c2numpy_writer *writer, const std::string name, c2numpy_type type) {
    writer->numColumns += 1;
    writer->columnNames.push_back(name);
    writer->columnTypes.push_back(type);
    return 0;
}

int c2numpy_open(c2numpy_writer *writer) {
    std::stringstream fileNameStream;
    fileNameStream << writer->outputFilePrefix;
    fileNameStream << writer->currentFileNumber;
    fileNameStream << ".npy";
    std::string fileName = fileNameStream.str();
    writer->file = fopen(fileName.c_str(), "wb");

    std::stringstream headerStream;
    headerStream << "{'descr': [";

    int column;
    for (column = 0;  column < writer->numColumns;  ++column) {
      headerStream << "('" << writer->columnNames[column] << "', '" << c2numpy_descr(writer->columnTypes[column]) << "')";
      if (column < writer->numColumns - 1)
        headerStream << ", ";
    }

    headerStream << "], 'fortran_order': False, 'shape': (";

    writer->sizeSeekPosition = headerStream.str().size();

    headerStream << writer->numRowsPerFile;

    writer->sizeSeekSize = headerStream.str().size() - writer->sizeSeekPosition;

    headerStream << ",), }";

    int headerSize = headerStream.str().size();
    char version = 1;

    if (headerSize > 65535) version = 2;
    while ((6 + 2 + (version == 1 ? 2 : 4) + headerSize) % 16 != 0) {
      headerSize += 1;
      headerStream << " ";
      if (headerSize > 65535) version = 2;
    }

    fwrite("\x93NUMPY", 1, 6, writer->file);
    if (version == 1) {
      fwrite("\x01\x00", 1, 2, writer->file);
      fwrite(&headerSize, 1, 2, writer->file);
      writer->sizeSeekPosition += 6 + 2 + 2;
    }
    else {
      fwrite("\x02\x00", 1, 2, writer->file);
      fwrite(&headerSize, 1, 4, writer->file);
      writer->sizeSeekPosition += 6 + 2 + 4;
    }

    std::string header = headerStream.str();
    fwrite(header.c_str(), 1, header.size(), writer->file);

    return 0;
}

#define C2NUMPY_CHECK_ITEM {                                                    \
    if (writer->file == NULL) {                                                 \
        int status = c2numpy_open(writer);                                      \
        if (status != 0)                                                        \
            return status;                                                      \
    }                                                                           \
}

#define C2NUMPY_INCREMENT_ITEM {                                                \
    if (writer->currentColumn == 0) {                                           \
        writer->currentRowInFile += 1;                                          \
        if (writer->currentRowInFile == writer->numRowsPerFile) {               \
            fclose(writer->file);                                               \
            writer->file = NULL;                                                \
            writer->currentRowInFile = 0;                                       \
            writer->currentFileNumber += 1;                                     \
        }                                                                       \
    }                                                                           \
    return 0;                                                                   \
}

int c2numpy_bool(c2numpy_writer *writer, int8_t data) {   // "bool" is just a byte
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_BOOL) return -1;
    fwrite(&data, sizeof(int8_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_int(c2numpy_writer *writer, int64_t data) {   // Numpy's default int is 64-bit
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT) return -1;
    fwrite(&data, sizeof(int64_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_intc(c2numpy_writer *writer, int data) {      // the built-in C int
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTC) return -1;
    fwrite(&data, sizeof(int), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_intp(c2numpy_writer *writer, size_t data) {   // intp is Numpy's way of saying size_t
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTP) return -1;
    fwrite(&data, sizeof(size_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_int8(c2numpy_writer *writer, int8_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT8) return -1;
    fwrite(&data, sizeof(int8_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_int16(c2numpy_writer *writer, int16_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT16) return -1;
    fwrite(&data, sizeof(int16_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_int32(c2numpy_writer *writer, int32_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT32) return -1;
    fwrite(&data, sizeof(int32_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_int64(c2numpy_writer *writer, int64_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT64) return -1;
    fwrite(&data, sizeof(int64_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_uint8(c2numpy_writer *writer, uint8_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT8) return -1;
    fwrite(&data, sizeof(uint8_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_uint16(c2numpy_writer *writer, uint16_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT16) return -1;
    fwrite(&data, sizeof(uint16_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_uint32(c2numpy_writer *writer, uint32_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT32) return -1;
    fwrite(&data, sizeof(uint32_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_uint64(c2numpy_writer *writer, uint64_t data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT64) return -1;
    fwrite(&data, sizeof(uint64_t), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_float(c2numpy_writer *writer, double data) {   // Numpy's "float" is a double
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT) return -1;
    fwrite(&data, sizeof(double), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

// int c2numpy_float16(c2numpy_writer *writer, ??? data) {   // how to do float16 in C?
//     C2NUMPY_CHECK_ITEM
//     if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT16) return -1;
//     fwrite(&data, sizeof(???), 1, writer->file);
//     writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
//     C2NUMPY_INCREMENT_ITEM
// }

int c2numpy_float32(c2numpy_writer *writer, float data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT32) return -1;
    fwrite(&data, sizeof(float), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_float64(c2numpy_writer *writer, double data) {
    C2NUMPY_CHECK_ITEM
    if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT64) return -1;
    fwrite(&data, sizeof(double), 1, writer->file);
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
    C2NUMPY_INCREMENT_ITEM
}

// int c2numpy_complex(c2numpy_writer *writer, ??? data) {    // how to do complex in C?
//     C2NUMPY_CHECK_ITEM
//     if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX) return -1;
//     fwrite(&data, sizeof(???), 1, writer->file);
//     writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
//     C2NUMPY_INCREMENT_ITEM
// }

// int c2numpy_complex64(c2numpy_writer *writer, ??? data) {
//     C2NUMPY_CHECK_ITEM
//     if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX64) return -1;
//     fwrite(&data, sizeof(???), 1, writer->file);
//     writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
//     C2NUMPY_INCREMENT_ITEM
// }

// int c2numpy_complex128(c2numpy_writer *writer, ??? data) {
//     C2NUMPY_CHECK_ITEM
//     if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX128) return -1;
//     fwrite(&data, sizeof(???), 1, writer->file);
//     writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;
//     C2NUMPY_INCREMENT_ITEM
// }

int c2numpy_string(c2numpy_writer *writer, const char *data) {
    C2NUMPY_CHECK_ITEM

    int stringlength = writer->columnTypes[writer->currentColumn] - C2NUMPY_STRING;
    if (0 < stringlength  &&  stringlength < 155)
        fwrite(data, 1, stringlength, writer->file);
    else
        return -1;
    writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns;

    C2NUMPY_INCREMENT_ITEM
}

int c2numpy_close(c2numpy_writer *writer) {
    if (writer->file != NULL) {
        // we wrote fewer rows than we promised
        if (writer->currentRowInFile < writer->numRowsPerFile) {
            // so go back to the part of the header where that was written
            fseek(writer->file, writer->sizeSeekPosition, SEEK_SET);
            // overwrite it with spaces
            int i;
            for (i = 0;  i < writer->sizeSeekSize;  ++i)
                fputc(' ', writer->file);
            // now go back and write it again (it MUST be fewer or an equal number of digits)
            fseek(writer->file, writer->sizeSeekPosition, SEEK_SET);
            fprintf(writer->file, "%d", writer->currentRowInFile);
        }
        // now close it
        fclose(writer->file);
    }

    return 0;
}

#endif // C2NUMPY