path: root/src/H5Z.c

/*
 * Copyright (C) 1998 NCSA
 *                    All rights reserved.
 *
 * Programmer:  Robb Matzke <matzke@llnl.gov>
 *              Thursday, April 16, 1998
 *
 * Purpose:	Functions for data filters.
 */
#include <H5private.h>
#include <H5Eprivate.h>
#include <H5MMprivate.h>
#include <H5Oprivate.h>
#include <H5Zprivate.h>

#ifdef HAVE_ZLIB_H
#   include <zlib.h>
#endif

/* Interface initialization */
#define PABLO_MASK	H5Z_mask
#define INTERFACE_INIT H5Z_init_interface
static intn interface_initialize_g = FALSE;
static herr_t H5Z_init_interface (void);
static void H5Z_term_interface (void);

/*
 * The filter table maps filter identification numbers to structs that
 * contain a pointers to the filter function and timing statistics.
 */
typedef struct H5Z_class_t {
    H5Z_filter_t id;		/*filter ID number			*/
    char	*comment;	/*comment for debugging			*/
    H5Z_func_t	func;		/*the filter function			*/

#ifdef H5Z_DEBUG
    struct {
	hsize_t	total;		/*total number of bytes processed	*/
	hsize_t	errors;		/*bytes of total attributable to errors	*/
	H5_timer_t timer;	/*execution time including errors	*/
    } stats[2];			/*0=output, 1=input			*/
#endif
} H5Z_class_t;

static size_t		H5Z_table_alloc_g = 0;
static size_t		H5Z_table_used_g = 0;
static H5Z_class_t	*H5Z_table_g = NULL;

static H5Z_class_t *H5Z_find(H5Z_filter_t id);

/* Predefined filters */
static size_t H5Z_filter_deflate(uintn flags, size_t cd_nelmts,
				 const uintn cd_values[], size_t nbytes,
				 size_t *buf_size, void **buf);


/*-------------------------------------------------------------------------
 * Function:	H5Z_init_interface
 *
 * Purpose:	Initializes the data filter layer.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Thursday, April 16, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5Z_init_interface (void)
{
    FUNC_ENTER (H5Z_init_interface, FAIL);
    H5_add_exit (H5Z_term_interface);

    H5Z_register (H5Z_FILTER_DEFLATE, "deflate",
		  H5Z_filter_deflate);

    FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_term_interface
 *
 * Purpose:	Terminate the H5Z layer.
 *
 * Return:	void
 *
 * Programmer:	Robb Matzke
 *              Thursday, April 16, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static void
H5Z_term_interface (void)
{
    size_t	i;
    
#ifdef H5Z_DEBUG
    int		dir, nprint=0;
    char	comment[16], bandwidth[32];

    for (i=0; i<=H5Z_table_used_g; i++) {
	for (dir=0; dir<2; dir++) {
	    if (0==H5Z_table_g[i].stats[dir].total) continue;

	    if (0==nprint++) {
		/* Print column headers */
		HDfprintf (stderr, "H5Z: filter statistics accumulated "
			   "over life of library:\n");
		HDfprintf (stderr, "   %-16s %10s %10s %8s %8s %8s %10s\n",
			   "Filter", "Total", "Errors", "User",
			   "System", "Elapsed", "Bandwidth");
		HDfprintf (stderr, "   %-16s %10s %10s %8s %8s %8s %10s\n",
			   "------", "-----", "------", "----",
			   "------", "-------", "---------");
	    }

	    /* Truncate the comment to fit in the field */
	    strncpy(comment, H5Z_table_g[i].comment, sizeof comment);
	    comment[sizeof(comment)-1] = '\0';

	    /*
	     * Format bandwidth to have four significant digits and units
	     * of `B/s', `kB/s', `MB/s', `GB/s', or `TB/s' or the word
	     * `Inf' if the elapsed time is zero.
	     */
	    H5_bandwidth(bandwidth,
			 (double)(H5Z_table_g[i].stats[dir].total),
			 H5Z_table_g[i].stats[dir].timer.etime);

	    /* Print the statistics */
	    HDfprintf (stderr,
		       "   %c%-15s %10Hd %10Hd %8.2f %8.2f %8.2f "
		       "%10s\n", dir?'<':'>', comment, 
		       H5Z_table_g[i].stats[dir].total,
		       H5Z_table_g[i].stats[dir].errors,
		       H5Z_table_g[i].stats[dir].timer.utime,
		       H5Z_table_g[i].stats[dir].timer.stime,
		       H5Z_table_g[i].stats[dir].timer.etime,
		       bandwidth);
	}
    }
#endif

    /* Free the table */
    for (i=0; i<H5Z_table_used_g; i++) {
	H5MM_xfree(H5Z_table_g[i].comment);
    }
    H5Z_table_g = H5MM_xfree(H5Z_table_g);
    H5Z_table_used_g = H5Z_table_alloc_g = 0;
}


/*-------------------------------------------------------------------------
 * Function:	H5Zregister
 *
 * Purpose:	This function registers new filter. The COMMENT argument is
 *		used for debugging and may be the null pointer.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Thursday, April 16, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
herr_t
H5Zregister (H5Z_filter_t id, const char *comment, H5Z_func_t func)
{
    FUNC_ENTER (H5Zregister, FAIL);
    H5TRACE3("e","Zfsx",id,comment,func);

    /* Check args */
    if (id<0 || id>H5Z_FILTER_MAX) {
	HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
		       "invalid filter identification number");
    }
    if (id<256) {
	HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
		       "unable to modify predefined filters");
    }
    if (!func) {
	HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
		      "no function specified");
    }

    /* Do it */
    if (H5Z_register (id, comment, func)<0) {
	HRETURN_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL,
		       "unable to register filter");
    }

    FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_register
 *
 * Purpose:	Same as the public version except this one allows filters
 *		to be set for predefined method numbers <256
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Thursday, April 16, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
herr_t
H5Z_register (H5Z_filter_t id, const char *comment, H5Z_func_t func)
{
    size_t		i;
    
    FUNC_ENTER (H5Z_register, FAIL);

    assert (id>=0 && id<=H5Z_FILTER_MAX);

    /* Is the filter already registered? */
    for (i=0; i<H5Z_table_used_g; i++) {
	if (H5Z_table_g[i].id==id) break;
    }
    if (i>=H5Z_table_used_g) {
	if (H5Z_table_used_g>=H5Z_table_alloc_g) {
	    size_t n = MAX(32, 2*H5Z_table_alloc_g);
	    H5Z_class_t *table = H5MM_realloc(H5Z_table_g,
					      n*sizeof(H5Z_class_t));
	    if (!table) {
		HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
			      "unable to extend filter table");
	    }
	    H5Z_table_g = table;
	    H5Z_table_alloc_g = n;
	}

	/* Initialize */
	i = H5Z_table_used_g++;
	HDmemset(H5Z_table_g+i, 0, sizeof(H5Z_class_t));
	H5Z_table_g[i].id = id;
	H5Z_table_g[i].comment = H5MM_xstrdup(comment);
	H5Z_table_g[i].func = func;
    } else {
	/* Replace old contents */
	H5MM_xfree(H5Z_table_g[i].comment);
	H5Z_table_g[i].comment = H5MM_xstrdup(comment);
	H5Z_table_g[i].func = func;
    }

    FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_append
 *
 * Purpose:	Append another filter to the specified pipeline.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Tuesday, August  4, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
herr_t
H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, uintn flags,
	   size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/])
{
    size_t	idx, i;
    
    FUNC_ENTER(H5Z_append, FAIL);
    assert(pline);
    assert(filter>=0 && filter<=H5Z_FILTER_MAX);
    assert(0==(flags & ~H5Z_FLAG_DEFMASK));
    assert(0==cd_nelmts || cd_values);

    /*
     * Check filter limit.  We do it here for early warnings although we may
     * decide to relax this restriction in the future.
     */
    if (pline->nfilters>=32) {
	HRETURN_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL,
		      "too many filters in pipeline");
    }

    /* Allocate additional space in the pipeline if it's full */
    if (pline->nfilters>=pline->nalloc) {
	H5O_pline_t x;
	x.nalloc = MAX(32, 2*pline->nalloc);
	x.filter = H5MM_realloc(pline->filter, x.nalloc*sizeof(x.filter[0]));
	if (NULL==x.filter) {
	    HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
			  "memory allocation failed for filter pipeline");
	}
	pline->nalloc = x.nalloc;
	pline->filter = x.filter;
    }
    
    /* Add the new filter to the pipeline */
    idx = pline->nfilters;
    pline->filter[idx].id = filter;
    pline->filter[idx].flags = flags;
    pline->filter[idx].name = NULL; /*we'll pick it up later*/
    pline->filter[idx].cd_nelmts = cd_nelmts;
    if (cd_nelmts>0) {
	pline->filter[idx].cd_values = H5MM_malloc(cd_nelmts*sizeof(uintn));
	if (NULL==pline->filter[idx].cd_values) {
	    HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
			  "memory allocation failed for filter");
	}
	for (i=0; i<cd_nelmts; i++) {
	    pline->filter[idx].cd_values[i] = cd_values[i];
	}
    }
    pline->nfilters++;

    FUNC_LEAVE(SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_find
 *
 * Purpose:	Given a filter ID return a pointer to a global struct that
 *		defines the filter.
 *
 * Return:	Success:	Ptr to entry in global filter table.
 *
 *		Failure:	NULL
 *
 * Programmer:	Robb Matzke
 *              Wednesday, August  5, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static H5Z_class_t *
H5Z_find(H5Z_filter_t id)
{
    size_t	i;

    FUNC_ENTER(H5Z_find, NULL);

    for (i=0; i<H5Z_table_used_g; i++) {
	if (H5Z_table_g[i].id == id) {
	    return H5Z_table_g + i;
	}
    }

    FUNC_LEAVE(NULL);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_pipeline
 *
 * Purpose:	Process data through the filter pipeline.  The FLAGS argument
 *		is the filter invocation flags (definition flags come from
 *		the PLINE->filter[].flags).  The filters are processed in
 *		definition order unless the H5Z_FLAG_REVERSE is set.  The
 *		FILTER_MASK is a bit-mask to indicate which filters to skip
 *		and on exit will indicate which filters failed.  Each
 *		filter has an index number in the pipeline and that index
 *		number is the filter's bit in the FILTER_MASK.  NBYTES is the
 *		number of bytes of data to filter and on exit should be the
 *		number of resulting bytes while BUF_SIZE holds the total
 *		allocated size of the buffer, which is pointed to BUF.
 *
 *		If the buffer must grow during processing of the pipeline
 *		then the pipeline function should free the original buffer
 *		and return a fresh buffer, adjusting BUF_SIZE accordingly.
 *
 * Return:	Success:	0
 *
 *		Failure:	-1
 *
 * Programmer:	Robb Matzke
 *              Tuesday, August  4, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
herr_t
H5Z_pipeline(H5F_t *f, const H5O_pline_t *pline, uintn flags,
	     uintn *filter_mask/*in,out*/, size_t *nbytes/*in,out*/,
	     size_t *buf_size/*in,out*/, void **buf/*in,out*/)
{
    size_t	i, idx, new_nbytes;
    H5Z_class_t	*fclass=NULL;
    uintn	failed = 0;
#ifdef H5Z_DEBUG
    H5_timer_t	timer;
#endif
    
    FUNC_ENTER(H5Z_pipeline, FAIL);
    
    assert(f);
    assert(0==(flags & ~H5Z_FLAG_INVMASK));
    assert(filter_mask);
    assert(nbytes && *nbytes>0);
    assert(buf_size && *buf_size>0);
    assert(buf && *buf);
    assert(!pline || pline->nfilters<32);

    if (pline && (flags & H5Z_FLAG_REVERSE)) {
	for (i=pline->nfilters; i>0; --i) {
	    idx = i-1;
	    
	    if (*filter_mask & ((unsigned)1<<idx)) {
		failed |= (unsigned)1 << idx;
		continue;/*filter excluded*/
	    }
	    if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) {
		failed |= (unsigned)1 << idx;
		HRETURN_ERROR(H5E_PLINE, H5E_READERROR, FAIL,
			      "required filter is not registered");
	    }
#ifdef H5Z_DEBUG
	    H5_timer_begin(&timer);
#endif
	    new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags),
					pline->filter[idx].cd_nelmts,
					pline->filter[idx].cd_values,
					*nbytes, buf_size, buf);
#ifdef H5Z_DEBUG
	    H5_timer_end(&(fclass->stats[1].timer), &timer);
	    fclass->stats[1].total += MAX(*nbytes, new_nbytes);
	    if (0==new_nbytes) fclass->stats[1].errors += *nbytes;
#endif
	    if (0==new_nbytes) {
		failed |= (unsigned)1 << idx;
		HRETURN_ERROR(H5E_PLINE, H5E_READERROR, FAIL,
			      "filter returned failure");
	    }
	    *nbytes = new_nbytes;
	}
    } else if (pline) {
	for (idx=0; idx<pline->nfilters; idx++) {
	    if (*filter_mask & ((unsigned)1<<idx)) {
		failed |= (unsigned)1 << idx;
		continue; /*filter excluded*/
	    }
	    if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) {
		failed |= (unsigned)1 << idx;
		if (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) {
		    H5E_clear();
		    continue;
		} else {
		    HRETURN_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL,
				  "required filter is not registered");
		}
	    }
#ifdef H5Z_DEBUG
	    H5_timer_begin(&timer);
#endif
	    new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags),
					pline->filter[idx].cd_nelmts,
					pline->filter[idx].cd_values,
					*nbytes, buf_size, buf);
#ifdef H5Z_DEBUG
	    H5_timer_end(&(fclass->stats[0].timer), &timer);
	    fclass->stats[0].total += MAX(*nbytes, new_nbytes);
	    if (0==new_nbytes) fclass->stats[0].errors += *nbytes;
#endif
	    if (0==new_nbytes) {
		failed |= (unsigned)1 << idx;
		if (0==(pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) {
		    HRETURN_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL,
				  "filter returned failure");
		} else {
		    H5E_clear();
		}
	    } else {
		*nbytes = new_nbytes;
	    }
	}
    }

    *filter_mask = failed;
    FUNC_LEAVE(SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5Z_filter_deflate
 *
 * Purpose:	
 *
 * Return:	Success:	
 *
 *		Failure:	
 *
 * Programmer:	Robb Matzke
 *              Thursday, April 16, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static size_t
H5Z_filter_deflate (uintn flags, size_t cd_nelmts, const uintn cd_values[],
		    size_t nbytes, size_t *buf_size, void **buf)
{
    size_t	ret_value = 0;
    int		aggression = 6;
    void	*outbuf = NULL;
#if defined(HAVE_LIBZ) && defined(HAVE_ZLIB_H)
    int		status;
#endif
    
    FUNC_ENTER (H5Z_filter_deflate, 0);

    /* Get aggression level */
    if (cd_nelmts!=1 || cd_values[0]>9) {
	HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
		    "invalid deflate aggression level");
    }
    aggression = cd_values[0];

#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H)
    if (flags & H5Z_FLAG_REVERSE) {
	/* Input; uncompress */
	z_stream	z_strm;
	size_t		nalloc = *buf_size;

	outbuf = H5MM_malloc(nalloc);
	HDmemset(&z_strm, 0, sizeof(z_strm));
	z_strm.next_in = *buf;
	z_strm.avail_in = nbytes;
	z_strm.next_out = outbuf;
	z_strm.avail_out = nalloc;
	if (Z_OK!=inflateInit(&z_strm)) {
	    HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
	}
	while (1) {
	    status = inflate(&z_strm, Z_SYNC_FLUSH);
	    if (Z_STREAM_END==status) break;	/*done*/
	    if (Z_OK!=status) {
		inflateEnd(&z_strm);
		HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
	    }
	    if (Z_OK==status && 0==z_strm.avail_out) {
		nalloc *= 2;
		outbuf = H5MM_realloc(outbuf, nalloc);
		z_strm.next_out = (char*)outbuf + z_strm.total_out;
		z_strm.avail_out = nalloc - z_strm.total_out;
	    }
	}
	
	H5MM_xfree(*buf);
	*buf = outbuf;
	outbuf = NULL;
	*buf_size = nalloc;
	ret_value = z_strm.total_out;
	inflateEnd(&z_strm);
	
    } else {
	/*
	 * Output; compress but fail if the result would be larger than the
	 * input.  The library doesn't provide in-place compression, so we
	 * must allocate a separate buffer for the result.
	 */
	const Bytef	*z_src = (const Bytef*)(*buf);
	Bytef		*z_dst;		/*destination buffer		*/
	uLongf		z_dst_nbytes = (uLongf)nbytes;
	uLong		z_src_nbytes = (uLong)nbytes;

	if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) {
	    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
			"unable to allocate deflate destination buffer");
	}
	status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes,
			    aggression);
	if (Z_BUF_ERROR==status) {
	    HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow");
	} else if (Z_MEM_ERROR==status) {
	    HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate memory error");
	} else if (Z_OK!=status) {
	    HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate error");
	} else {
	    H5MM_xfree(*buf);
	    *buf = outbuf;
	    outbuf = NULL;
	    *buf_size = nbytes;
	    ret_value = z_dst_nbytes;
	}
    }
#else
    HGOTO_ERROR (H5E_PLINE, H5E_UNSUPPORTED, 0,
		 "hdf5 was not compiled with zlib-1.0.2 or better");
#endif

 done:
    H5MM_xfree(outbuf);
    FUNC_LEAVE (ret_value);
}