path: root/src/H5Znbit.c

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by the Board of Trustees of the University of Illinois.         *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the files COPYING and Copyright.html.  COPYING can be found at the root   *
 * of the source code distribution tree; Copyright.html can be found at the  *
 * root level of an installed copy of the electronic HDF5 document set and   *
 * is linked from the top-level documents page.  It can also be found at     *
 * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html.  If you do not have     *
 * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#define H5Z_PACKAGE		/*suppress error about including H5Zpkg	  */

/* Pablo information */
/* (Put before include files to avoid problems with inline functions) */
#define PABLO_MASK      H5Z_nbit_mask

#include "H5private.h"		/* Generic Functions			*/
#include "H5Eprivate.h"		/* Error handling		  	*/
#include "H5Fprivate.h"         /* File access                          */
#include "H5MMprivate.h"	/* Memory management			*/
#include "H5Oprivate.h"		/* Object headers		  	*/
#include "H5Ppublic.h"		/* Property lists			*/
#include "H5Tpublic.h"		/* Datatype functions			*/
#include "H5Zpkg.h"		/* Data filters				*/

#ifdef H5_HAVE_FILTER_NBIT

/* Local function prototypes */
static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static size_t H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], 
                              size_t nbytes, size_t *buf_size, void **buf);

typedef struct {
   int precision;
   int offset;
   int sizeof_datatype;
   int order;
} para;

void H5Z_nbit_next_char(int *j, int *buf_len);
void H5Z_nbit_byte_order(int *big_endian, int *little_endian);
void H5Z_nbit_decompress_one_byte(void *data, int i, int k, int begin_i, int end_i, char *buffer,
                         int *j, int *buf_len, para p);
void H5Z_nbit_compress_one_byte(void *data, int i, int k, int begin_i, int end_i, char *buffer, 
                       int *j, int *buf_len, para p);
void H5Z_nbit_decompress(void *data, int data_size, char *buffer, int buffer_size, para p);
void H5Z_nbit_compress(void *data, int data_size, char *buffer, int buffer_size, para p);

/* This message derives from H5Z */
H5Z_class_t H5Z_NBIT[1] = {{
    H5Z_CLASS_T_VERS,       /* H5Z_class_t version */
    H5Z_FILTER_NBIT,		/* Filter id number		*/
    1,              /* Assume encoder present: check before registering */
    1,                  /* decoder_present flag (set to true) */
    "nbit",			    /* Filter name for debugging	*/
    H5Z_can_apply_nbit,		/* The "can apply" callback     */
    H5Z_set_local_nbit,         /* The "set local" callback     */
    H5Z_filter_nbit,		/* The actual filter function	*/
}};

/* Local macros */
#define H5Z_NBIT_USER_NPARMS     0    /* Number of parameters that users can set */
#define H5Z_NBIT_TOTAL_NPARMS    5    /* Total number of parameters for filter */
#define H5Z_NBIT_PARM_PRECIS     0    /* "Local" parameter for datatype precision */
#define H5Z_NBIT_PARM_OFFSET     1    /* "Local" parameter for datatype offset */
#define H5Z_NBIT_PARM_TSIZE      2    /* "Local" parameter for datatype size */
#define H5Z_NBIT_PARM_NELMTS     3    /* "Local" parameter for number of elements in the chunk */
#define H5Z_NBIT_PARM_ORDER      4    /* "Local" parameter for datatype byte order */

/*-------------------------------------------------------------------------
 * Function:	H5Z_can_apply_nbit
 *
 * Purpose:	Check the parameters for nbit compression for validity and
 *              whether they fit a particular dataset.
 *
 * Note:        
 *              
 * Return:	Success: Non-negative
 *		Failure: Negative
 *
 * Programmer:  Quincey Koziol
 *              Monday, April  7, 2003 
 *
 * Modifications: 
 *              Xiaowen Wu
 *              Tuesday, December 21, 2004
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5Z_can_apply_nbit(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id)
{
    unsigned dtype_size;                /* Datatype's size (in bytes) */
    H5T_order_t dtype_order;            /* Datatype's endianness order */
    herr_t ret_value=TRUE;              /* Return value */

    FUNC_ENTER_NOAPI(H5Z_can_apply_nbit, FAIL)

    /* Get datatype's size, for checking the "datatype size" */
    if((dtype_size = H5Tget_size(type_id)) == 0)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")

    /* Get datatype's endianness order */
    if((dtype_order = H5Tget_order(type_id)) == H5T_ORDER_ERROR)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order")

    /* Range check datatype's endianness order */
    /* (Note: this may not handle non-atomic datatypes well) */
    if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
	HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype endianness order")

done:
    FUNC_LEAVE_NOAPI(ret_value)
} /* end H5Z_can_apply_nbit() */


/*-------------------------------------------------------------------------
 * Function:	H5Z_set_local_nbit
 *
 * Purpose:	Set the "local" dataset parameters for nbit compression.
 *
 * Return:	Success: Non-negative
 *		Failure: Negative
 *
 * Programmer:	Quincey Koziol
 *              Monday, April 7, 2003
 *
 * Modifications:
 *              Xiaowen Wu
 *              Tuesday, December 21, 2004 
 *                
 *-------------------------------------------------------------------------
 */
static herr_t
H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
    unsigned flags;         /* Filter flags */
    size_t cd_nelmts=H5Z_NBIT_USER_NPARMS;     /* Number of filter parameters */
    unsigned cd_values[H5Z_NBIT_TOTAL_NPARMS];  /* Filter parameters */
    hsize_t dims[H5O_LAYOUT_NDIMS];             /* Dataspace (i.e. chunk) dimensions */
    int ndims;                  /* Number of (chunk) dimensions */
    hssize_t npoints;           /* Number of points in the dataspace */
    H5T_order_t dtype_order;    /* Datatype's endianness order */
    int dtype_size;             /* Datatype's size (in bytes) */
    size_t dtype_precision;     /* Datatype's precision (in bits) */
    int dtype_offset;        /* Datatype's offset (in bits) */
    herr_t ret_value=SUCCEED;   /* Return value */

    FUNC_ENTER_NOAPI(H5Z_set_local_nbit, FAIL)

    /* Get the filter's current parameters */
#ifdef H5_WANT_H5_V1_6_COMPAT
    if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_NBIT,&flags,&cd_nelmts, cd_values,0,NULL)<0)
#else
    if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_NBIT,&flags,&cd_nelmts, cd_values,0,NULL,NULL)<0)
#endif
	HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get nbit parameters")

    /* Get datatype's size, for checking the "datatype size" */
    if((dtype_size=H5Tget_size(type_id))==0)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size");

    /* Set "local" parameter for this dataset's "datatype size" */
    cd_values[H5Z_NBIT_PARM_TSIZE]=dtype_size;

    /* Get datatype's precision, for checking the "datatype precision"  */
    if((dtype_precision=H5Tget_precision(type_id))==0)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype precision");

    /* Set "local" parameter for this dataset's "datatype precision" */
    cd_values[H5Z_NBIT_PARM_PRECIS]=dtype_precision;

    /* Get datatype's offset, for checking the "datatype offset"  */
    if((dtype_offset=H5Tget_offset(type_id))<0)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype offset");

    /* Set "local" parameter for this dataset's "datatype offset" */
    cd_values[H5Z_NBIT_PARM_OFFSET]=dtype_offset;

    /* Get dimensions for dataspace */
    if ((ndims=H5Sget_simple_extent_dims(space_id, dims, NULL))<0)
        HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions")
    assert(ndims>0);

    /* Get total number of elements in the chunk */
    if ((npoints=H5Sget_simple_extent_npoints(space_id))<0)
        HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")

    /* Set "local" parameter for this dataset's "number of elements" */
    cd_values[H5Z_NBIT_PARM_NELMTS] = npoints;

    /* Get datatype's endianness order */
    if((dtype_order=H5Tget_order(type_id))==H5T_ORDER_ERROR)
	HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")

    /* Set the correct endianness for nbit */
    /* (Note: this may not handle non-atomic datatypes well) */
    switch(dtype_order) {
        case H5T_ORDER_LE:      /* Little-endian byte order */
            cd_values[H5Z_NBIT_PARM_ORDER] = 10;
            break;
        case H5T_ORDER_BE:      /* Big-endian byte order */
            cd_values[H5Z_NBIT_PARM_ORDER] = 20;
            break;
        default:
            HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
    } /* end switch */

    /* Modify the filter's parameters for this dataset */
    if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_NBIT, flags, H5Z_NBIT_TOTAL_NPARMS, cd_values)<0)
	HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local nbit parameters")

done:
    FUNC_LEAVE_NOAPI(ret_value)
} /* end H5Z_set_local_nbit() */


/*-------------------------------------------------------------------------
 * Function:	H5Z_filter_nbit
 *
 * Purpose:	Implement an I/O filter for storing packed n-bit data
 *              
 * Return:	Success: Size of buffer filtered
 *		Failure: 0	
 *
 * Programmer:	Xiaowen Wu
 *              Tuesday, December 21, 2004
 *
 * Modifications:
 *              
 *-------------------------------------------------------------------------
 */
static size_t
H5Z_filter_nbit (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], 
    size_t nbytes, size_t *buf_size, void **buf)
{
    size_t ret_value = 0;       /* return value */
    size_t size_out  = 0;       /* size of output buffer */
    size_t d_nelmts    = 0;       /* number of elements in the chunk */
    unsigned char *outbuf = NULL;    /* pointer to new output buffer */
    para nbit_param;          /* nbit parameter block */

    FUNC_ENTER_NOAPI(H5Z_filter_nbit, 0)

    /* check arguments */
    if (cd_nelmts!=5)
	HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid nbit aggression level")

    /* copy the filter parameters into the nbit parameter block */
    H5_ASSIGN_OVERFLOW(nbit_param.precision,cd_values[H5Z_NBIT_PARM_PRECIS],unsigned,int);
    H5_ASSIGN_OVERFLOW(nbit_param.offset,cd_values[H5Z_NBIT_PARM_OFFSET],unsigned,int);
    H5_ASSIGN_OVERFLOW(nbit_param.sizeof_datatype,cd_values[H5Z_NBIT_PARM_TSIZE],unsigned,int);
    H5_ASSIGN_OVERFLOW(nbit_param.order,cd_values[H5Z_NBIT_PARM_ORDER],unsigned,int);

    /* copy a filter parameter to d_nelmts */
    H5_ASSIGN_OVERFLOW(d_nelmts,cd_values[H5Z_NBIT_PARM_NELMTS],unsigned,int);

    printf("\nprecision: %d", nbit_param.precision);
    printf("    offset: %d\n", nbit_param.offset);

    /* no need to compress or decompress 
    if (nbit_param.precision == nbit_param.sizeof_datatype * 8) { 
        ret_value = *buf_size;
        goto done;
    }*/

    /* input; decompress */
    if (flags & H5Z_FLAG_REVERSE) {
        size_out = d_nelmts * nbit_param.sizeof_datatype;

        /* allocate memory space for decompressed buffer */
        if(NULL==(outbuf = H5MM_malloc(size_out)))
            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for nbit decompression")
  
        /* decompress the buffer */
        H5Z_nbit_decompress(outbuf, d_nelmts, *buf, nbytes, nbit_param);   
    }
    /* output; compress */
    else {
        if((d_nelmts * nbit_param.precision) % (sizeof(char) * 8) != 0)
            size_out  = (d_nelmts * nbit_param.precision) / (sizeof(char) * 8) + 1;
        else
            size_out  = (d_nelmts * nbit_param.precision) / (sizeof(char) * 8);

        /* allocate memory space for compressed buffer */
        if(NULL==(outbuf = H5MM_malloc(size_out)))
            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for nbit compression")
  
        /* compress the buffer */
        H5Z_nbit_compress(*buf, d_nelmts, outbuf, size_out, nbit_param);
    }

    /* free the input buffer */
    H5MM_xfree(*buf);

    /* set return values */
    *buf = outbuf;
    outbuf = NULL;
    *buf_size = size_out;
    ret_value = size_out;

done: 
    if(outbuf)
        H5MM_xfree(outbuf);
    FUNC_LEAVE_NOAPI(ret_value)
}

/* assume one byte has 8 bit, padding bit is 0
   assume size of char is one byte
   assume one item of certain data type is stored continously in bytes 
   data type regardless what is treated on byte basis 
*/

void H5Z_nbit_next_char(int *j, int *buf_len)
{
   ++(*j); 
   *buf_len = 8 * sizeof(char);
}

void H5Z_nbit_byte_order(int *big_endian, int *little_endian)
{
   /* decide the machine's byte order */
   union {
      long l; 
      char c[sizeof(long)];
   } u;
   u.l = 1;
   if(u.c[0] == 1) 
      *little_endian = 1;
   else if(u.c[sizeof(long) - 1] == 1) 
      *big_endian = 1;
   else {
      printf("Abort: unkown byte order!\n");
      exit(1);
   }
}       

void H5Z_nbit_decompress_one_byte(void *data, int i, int k, int begin_i, int end_i, char *buffer, 
            int *j, int *buf_len, para p)
{
   int dat_len; /* dat_len is the number of bits to be copied in each data char */
   int datatype_len, char_offset;
   char val; /* value to be copied in each data char */

   datatype_len = p.sizeof_datatype * 8;

   /* initialize value and bits of char to be copied */
   val = buffer[*j];
   char_offset = 0;
   if(begin_i != end_i) { /* significant bits occupy >1 char */
      if(k == begin_i) 
         dat_len = 8 - (datatype_len - p.precision - p.offset) % 8;
      else if(k == end_i) {
         if(p.offset % 8 == 0)
            dat_len = 8;
         else
            dat_len = (datatype_len - p.offset) % 8;
         char_offset = 8 - dat_len;
      }
      else
         dat_len = 8;
   } else { /* all significant bits in one char */
      if(p.offset % 8 != 0)
         char_offset = p.offset % 8;
      dat_len = p.precision;
   }

   if(*buf_len > dat_len) { 
      ((char *)data)[i * p.sizeof_datatype + k] |=
      ((val >> (*buf_len - dat_len)) & ~(~0 << dat_len)) << char_offset;
      *buf_len -= dat_len;
   } else {
      ((char *)data)[i * p.sizeof_datatype + k] |=
      ((val & ~(~0 << *buf_len)) << (dat_len - *buf_len)) << char_offset;
      dat_len -= *buf_len;
      H5Z_nbit_next_char(j, buf_len);
      if(dat_len == 0) 
         return;
      val = buffer[*j];
   
      ((char *)data)[i * p.sizeof_datatype + k] |=
      ((val >> (*buf_len - dat_len)) & ~(~0 << dat_len)) << char_offset;
      *buf_len -= dat_len; 
   }
}

/* decompress buffer to original data form */
void H5Z_nbit_decompress(void *data, int data_size, char *buffer, int buffer_size, para p) 
{
   /* i: index of data, j: index of buffer, 
      buf_len: number of bits to be copied in current char */
   int i, j, buf_len; 
   /* begin_i: the index of byte having first significant bit
      end_i: the index of byte having last significant bit */
   int k, begin_i, end_i, big_endian, little_endian, datatype_len;

   printf("decompress is called \n");

   /* may not have to initialize to zeros */
   for(i = 0; i < data_size * p.sizeof_datatype; i++)  
      ((char *)data)[i] = 0; 
   
   /* decide the byte order of the machine */
   little_endian = big_endian = 0;
   H5Z_nbit_byte_order(&big_endian, &little_endian);

   datatype_len = p.sizeof_datatype * 8;

   /* initialization before the loop */ 
   j = 0; 
   buf_len = sizeof(char) * 8; 

   if(p.order == 10) { /* little endian */
      printf("\ndatatype is little-endian\n");
      /* calculate begin_i and end_i */
      if((p.precision + p.offset) % 8 != 0)
         begin_i = (p.precision + p.offset) / 8;
      else
         begin_i = (p.precision + p.offset) / 8 - 1;
      end_i = p.offset / 8;

      for(i = 0; i < data_size; i++) 
         for(k = begin_i; k >= end_i; k--)
            H5Z_nbit_decompress_one_byte(data, i, k, begin_i, end_i, buffer, &j, &buf_len, p);
   }

   if(p.order == 20) { /* big endian */
      printf("\ndatatype is big-endian\n");
      /* calculate begin_i and end_i */
      begin_i = (datatype_len - p.precision - p.offset) / 8;
      if(p.offset % 8 != 0)
         end_i = (datatype_len - p.offset) / 8;
      else
         end_i = (datatype_len - p.offset) / 8 - 1;

      for(i = 0; i < data_size; i++) 
         for(k = begin_i; k <= end_i; k++)
            H5Z_nbit_decompress_one_byte(data, i, k, begin_i, end_i, buffer, &j, &buf_len, p);
   } 
}

void H5Z_nbit_compress_one_byte(void *data, int i, int k, int begin_i, int end_i, char *buffer, 
                       int *j, int *buf_len, para p)
{
   int dat_len; /* dat_len is the number of bits to be copied in each data char */
   int datatype_len;
   char val; /* value to be copied in each data char */

   datatype_len = p.sizeof_datatype * 8;

   /* initialize value and bits of char to be copied */
   val = ((char *)data)[i * p.sizeof_datatype + k];
   if(begin_i != end_i) { /* significant bits occupy >1 char */
      if(k == begin_i) 
         dat_len = 8 - (datatype_len - p.precision - p.offset) % 8;
      else if(k == end_i) {
         if(p.offset % 8 == 0)
            dat_len = 8;
         else
            dat_len = (datatype_len - p.offset) % 8;
         val >>= 8 - dat_len;
      }
      else
         dat_len = 8;
   } else { /* all significant bits in one char */
      if(p.offset % 8 != 0)
         val >>= p.offset % 8;
      dat_len = p.precision;
   }

   if(*buf_len > dat_len) { 
      buffer[*j] |= (val & ~(~0 << dat_len)) << (*buf_len - dat_len);
      *buf_len -= dat_len;
   } else {
      buffer[*j] |= (val >> (dat_len - *buf_len)) & ~(~0 << *buf_len);
      dat_len -= *buf_len;
      H5Z_nbit_next_char(j, buf_len);
      if(dat_len == 0) 
         return;
      val &= ~(~0 << dat_len);
   
      buffer[*j] |= (val & ~(~0 << dat_len)) << (*buf_len - dat_len);
      *buf_len -= dat_len; 
   }
}

/* copy array of certain data type to buffer in compressed form
   return number of unused bits in the buffer */
void H5Z_nbit_compress(void *data, int data_size, char *buffer, int buffer_size, para p) {
   /* i: index of data, j: index of buffer, 
      buf_len: number of bits to be filled in current char */
   int i, j, buf_len; 
   /* begin_i: the index of byte having first significant bit
      end_i: the index of byte having last significant bit */ 
   int k, begin_i, end_i, big_endian, little_endian, datatype_len;

   printf("compress is called \n");

   for(j = 0; j < buffer_size; j++) /* initialize buffer to be zeros */ 
      buffer[j] = 0;

   /* decide the byte order of the machine */
   big_endian = little_endian = 0;
   H5Z_nbit_byte_order(&big_endian, &little_endian);

   datatype_len = p.sizeof_datatype * 8;
   /* initialization before the loop */ 
   j = 0; 
   buf_len = sizeof(char) * 8; 

   if(p.order == 10) { /* little endian */
      printf("\ndatatype is little-endian\n");
      /* calculate begin_i and end_i */
      if((p.precision + p.offset) % 8 != 0)
         begin_i = (p.precision + p.offset) / 8;
      else
         begin_i = (p.precision + p.offset) / 8 - 1;
      end_i = p.offset / 8;

      for(i = 0; i < data_size; i++) 
         for(k = begin_i; k >= end_i; k--) 
            H5Z_nbit_compress_one_byte(data, i, k, begin_i, end_i, buffer, &j, &buf_len, p);
   }

   if(p.order == 20) { /* big endian */
      printf("\ndatatype is big-endian\n");
      /* calculate begin_i and end_i */
      begin_i = (datatype_len - p.precision - p.offset) / 8;
      if(p.offset % 8 != 0)
         end_i = (datatype_len - p.offset) / 8;
      else
         end_i = (datatype_len - p.offset) / 8 - 1;

      for(i = 0; i < data_size; i++) 
         for(k = begin_i; k <= end_i; k++) 
            H5Z_nbit_compress_one_byte(data, i, k, begin_i, end_i, buffer, &j, &buf_len, p);
   }    
}
#endif /* H5_HAVE_FILTER_NZIP */