summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorQuincey Koziol <koziol@hdfgroup.org>2002-11-20 13:15:18 (GMT)
committerQuincey Koziol <koziol@hdfgroup.org>2002-11-20 13:15:18 (GMT)
commitac48a23e2b72b03d60d4fdf10957ed31beb5b70d (patch)
treeffa3319c694f416086419118f3fc3bc3b1e6fca1
parent892cc41777c9a104e49ad6d9bac21f085dba6b95 (diff)
downloadhdf5-ac48a23e2b72b03d60d4fdf10957ed31beb5b70d.zip
hdf5-ac48a23e2b72b03d60d4fdf10957ed31beb5b70d.tar.gz
hdf5-ac48a23e2b72b03d60d4fdf10957ed31beb5b70d.tar.bz2
[svn-r6114] Purpose:
Code Cleanup & New Feature Description: H5config.h.in: Removed H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags. Added H5_HAVE_FILTER_DEFLATE flag. H5Z.c: H5Zprivate.h: H5Zpublic.h: Switched from using H5_HAVE_COMPRESSION flag in favor of H5_HAVE_FILTER_DEFLATE. Added H5Zunregister & H5Zfilter_avail API functions. Changed a numeric constant (256) to a symbolic constant (H5Z_FILTER_RESERVED). Automatically add the shuffling filter to the list of available filters (when it is enabled). Moved prototypes for H5Z_filter_deflate & H5Z_filter_shuffle from the public header into the private header. H5Zdeflate.c: Switched from using H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags in favor of H5_HAVE_FILTER_DEFLATE. Cleaned up formatting & error reporting a bit. H5Zshuffle.c: Rewrote shuffling algorithm to be more efficient. Added error checking & reporting. Added standard Pablo information. Added standard function header comment. Added FUNC_ENTER & FUNC_LEAVE macros. Platforms tested: Tested h5committest {arabica (fortran), eirene (fortran, C++) modi4 (parallel, fortran)} FreeBSD 4.7 (sleipnir)
-rw-r--r--src/H5Z.c133
-rw-r--r--src/H5Zdeflate.c55
-rw-r--r--src/H5Zprivate.h9
-rw-r--r--src/H5Zpublic.h11
-rw-r--r--src/H5Zshuffle.c153
-rw-r--r--src/H5config.h.in7
6 files changed, 253 insertions, 115 deletions
diff --git a/src/H5Z.c b/src/H5Z.c
index 4016481..9c240c3 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -19,10 +19,12 @@
static int interface_initialize_g = 0;
static herr_t H5Z_init_interface (void);
+/* Local variables */
static size_t H5Z_table_alloc_g = 0;
static size_t H5Z_table_used_g = 0;
static H5Z_class_t *H5Z_table_g = NULL;
+/* Local functions */
/*-------------------------------------------------------------------------
@@ -44,9 +46,12 @@ H5Z_init_interface (void)
{
FUNC_ENTER_NOINIT(H5Z_init_interface);
-#ifdef H5_HAVE_FILTER_GZIP
+#ifdef H5_HAVE_FILTER_DEFLATE
H5Z_register (H5Z_FILTER_DEFLATE, "deflate", H5Z_filter_deflate);
-#endif /* H5_HAVE_FILTER_GZIP */
+#endif /* H5_HAVE_FILTER_DEFLATE */
+#ifdef H5_HAVE_FILTER_SHUFFLE
+ H5Z_register (H5Z_FILTER_SHUFFLE, "shuffle", H5Z_filter_shuffle);
+#endif /* H5_HAVE_FILTER_SHUFFLE */
FUNC_LEAVE (SUCCEED);
}
@@ -162,7 +167,7 @@ H5Zregister(H5Z_filter_t id, const char *comment, H5Z_func_t func)
/* Check args */
if (id<0 || id>H5Z_FILTER_MAX)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
- if (id<256)
+ if (id<H5Z_FILTER_RESERVED)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "unable to modify predefined filters");
if (!func)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no function specified");
@@ -180,7 +185,7 @@ done:
* Function: H5Z_register
*
* Purpose: Same as the public version except this one allows filters
- * to be set for predefined method numbers <256
+ * to be set for predefined method numbers <H5Z_FILTER_RESERVED
*
* Return: Non-negative on success/Negative on failure
*
@@ -235,6 +240,126 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5Zunregister
+ *
+ * Purpose: This function unregisters a filter.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, November 14, 2002
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Zunregister(H5Z_filter_t id)
+{
+ herr_t ret_value=SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(H5Zunregister, FAIL);
+ H5TRACE1("e","Zf",id);
+
+ /* Check args */
+ if (id<0 || id>H5Z_FILTER_MAX)
+ HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
+ if (id<H5Z_FILTER_RESERVED)
+ HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "unable to modify predefined filters");
+
+ /* Do it */
+ if (H5Z_unregister (id)<0)
+ HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to unregister filter");
+
+done:
+ FUNC_LEAVE (ret_value);
+} /* end H5Zunregister() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_unregister
+ *
+ * Purpose: Same as the public version except this one allows filters
+ * to be unset for predefined method numbers <H5Z_FILTER_RESERVED
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, November 14, 2002
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Z_unregister (H5Z_filter_t id)
+{
+ size_t i; /* Local index variable */
+ herr_t ret_value=SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOINIT(H5Z_unregister);
+
+ assert (id>=0 && id<=H5Z_FILTER_MAX);
+
+ /* Is the filter already registered? */
+ for (i=0; i<H5Z_table_used_g; i++)
+ if (H5Z_table_g[i].id==id)
+ break;
+
+ /* Fail if filter not found */
+ if (i>=H5Z_table_used_g)
+ HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter is not registered");
+
+ /* Remove filter from table */
+ /* Don't worry about shrinking table size (for now) */
+ HDmemmove(&H5Z_table_g[i],&H5Z_table_g[i+1],sizeof(H5Z_class_t)*((H5Z_table_used_g-1)-i));
+ H5Z_table_used_g--;
+
+done:
+ FUNC_LEAVE (ret_value);
+} /* end H5Z_unregister() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Zfilter_avail
+ *
+ * Purpose: Check if a filter is available
+ *
+ * Return: Non-negative (TRUE/FALSE) on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, November 14, 2002
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5Zfilter_avail(H5Z_filter_t id)
+{
+ size_t i; /* Local index variable */
+ htri_t ret_value=FALSE; /* Return value */
+
+ FUNC_ENTER_API(H5Zfilter_avail, FAIL);
+ H5TRACE1("b","Zf",id);
+
+ /* Check args */
+ if(id<0 || id>H5Z_FILTER_MAX)
+ HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
+
+ /* Is the filter already registered? */
+ for(i=0; i<H5Z_table_used_g; i++)
+ if(H5Z_table_g[i].id==id) {
+ ret_value=TRUE;
+ break;
+ } /* end if */
+
+done:
+ FUNC_LEAVE (ret_value);
+} /* end H5Zfilter_avail() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5Z_append
*
* Purpose: Append another filter to the specified pipeline.
diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c
index b66012d..09cd7a0 100644
--- a/src/H5Zdeflate.c
+++ b/src/H5Zdeflate.c
@@ -10,13 +10,10 @@
#include "H5MMprivate.h"
#include "H5Zprivate.h"
-#ifdef H5_HAVE_FILTER_GZIP
+#ifdef H5_HAVE_FILTER_DEFLATE
#ifdef H5_HAVE_ZLIB_H
# include "zlib.h"
-#else
-/* Make sure compression is disabled too. */
-#undef H5_HAVE_COMPRESSION
#endif
/* Interface initialization */
@@ -28,7 +25,8 @@ static int interface_initialize_g = 0;
/*-------------------------------------------------------------------------
* Function: H5Z_filter_deflate
*
- * Purpose:
+ * Purpose: Implement an I/O filter around the 'deflate' algorithm in
+ * libz
*
* Return: Success:
*
@@ -42,53 +40,40 @@ static int interface_initialize_g = 0;
*-------------------------------------------------------------------------
*/
size_t
-#if defined(H5_HAVE_COMPRESSION)
H5Z_filter_deflate (unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
-#else
-H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
- const unsigned cd_values[], size_t UNUSED nbytes,
- size_t * UNUSED buf_size, void ** UNUSED buf)
-#endif
{
size_t ret_value = 0;
void *outbuf = NULL;
-#if defined(H5_HAVE_COMPRESSION)
int aggression = 6;
int status;
-#endif
FUNC_ENTER_NOAPI(H5Z_filter_deflate, 0);
/* Check arguments */
- if (cd_nelmts!=1 || cd_values[0]>9) {
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
- "invalid deflate aggression level");
- }
+ if (cd_nelmts!=1 || cd_values[0]>9)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level");
-#if defined(H5_HAVE_COMPRESSION)
aggression = cd_values[0];
if (flags & H5Z_FLAG_REVERSE) {
/* Input; uncompress */
z_stream z_strm;
size_t nalloc = *buf_size;
- if (NULL==(outbuf = H5MM_malloc(nalloc))) {
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
- "memory allocation failed for deflate uncompression");
- }
+ if (NULL==(outbuf = H5MM_malloc(nalloc)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression");
HDmemset(&z_strm, 0, sizeof(z_strm));
z_strm.next_in = *buf;
H5_ASSIGN_OVERFLOW(z_strm.avail_in,nbytes,size_t,uInt);
z_strm.next_out = outbuf;
H5_ASSIGN_OVERFLOW(z_strm.avail_out,nalloc,size_t,uInt);
- if (Z_OK!=inflateInit(&z_strm)) {
+ if (Z_OK!=inflateInit(&z_strm))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
- }
while (1) {
status = inflate(&z_strm, Z_SYNC_FLUSH);
- if (Z_STREAM_END==status) break; /*done*/
+ if (Z_STREAM_END==status)
+ break; /*done*/
if (Z_OK!=status) {
inflateEnd(&z_strm);
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
@@ -97,9 +82,7 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
nalloc *= 2;
if (NULL==(outbuf = H5MM_realloc(outbuf, nalloc))) {
inflateEnd(&z_strm);
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
- "memory allocation failed for deflate "
- "uncompression");
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression");
}
z_strm.next_out = (unsigned char*)outbuf + z_strm.total_out;
z_strm.avail_out = (uInt)(nalloc - z_strm.total_out);
@@ -112,7 +95,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
*buf_size = nalloc;
ret_value = z_strm.total_out;
inflateEnd(&z_strm);
-
} else {
/*
* Output; compress but fail if the result would be larger than the
@@ -124,12 +106,9 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
uLongf z_dst_nbytes = (uLongf)nbytes;
uLong z_src_nbytes = (uLong)nbytes;
- if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) {
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
- "unable to allocate deflate destination buffer");
- }
- status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes,
- aggression);
+ if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate deflate destination buffer");
+ status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
if (Z_BUF_ERROR==status) {
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow");
} else if (Z_MEM_ERROR==status) {
@@ -144,10 +123,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
ret_value = z_dst_nbytes;
}
}
-#else
- HGOTO_ERROR (H5E_PLINE, H5E_UNSUPPORTED, 0,
- "hdf5 was not compiled with zlib-1.0.2 or better");
-#endif
done:
if(outbuf)
@@ -155,4 +130,4 @@ done:
FUNC_LEAVE (ret_value);
}
-#endif /* H5_HAVE_FILTER_GZIP */
+#endif /* H5_HAVE_FILTER_DEFLATE */
diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h
index 0a20b13..388f204 100644
--- a/src/H5Zprivate.h
+++ b/src/H5Zprivate.h
@@ -33,6 +33,7 @@ struct H5O_pline_t; /*forward decl*/
H5_DLL herr_t H5Z_register(H5Z_filter_t id, const char *comment,
H5Z_func_t filter);
+H5_DLL herr_t H5Z_unregister (H5Z_filter_t id);
H5_DLL herr_t H5Z_append(struct H5O_pline_t *pline, H5Z_filter_t filter,
unsigned flags, size_t cd_nelmts,
const unsigned int cd_values[]);
@@ -43,4 +44,12 @@ H5_DLL herr_t H5Z_pipeline(H5F_t *f, const struct H5O_pline_t *pline,
H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id);
+/* Filter routines */
+size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts,
+ const unsigned cd_values[], size_t nbytes,
+ size_t *buf_size, void **buf);
+
+size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
+ const unsigned cd_values[], size_t nbytes,
+ size_t *buf_size, void **buf);
#endif
diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h
index 46f74a3..0b9a0bf 100644
--- a/src/H5Zpublic.h
+++ b/src/H5Zpublic.h
@@ -20,6 +20,7 @@ typedef int H5Z_filter_t;
#define H5Z_FILTER_NONE 0 /*reserved indefinitely */
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
#define H5Z_FILTER_SHUFFLE 2 /* shuffle the data */
+#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved */
#define H5Z_FILTER_MAX 65535 /*maximum filter id */
/* Flags for filter definition */
@@ -56,13 +57,11 @@ extern "C" {
H5_DLL herr_t H5Zregister(H5Z_filter_t id, const char *comment,
H5Z_func_t filter);
-size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts,
- const unsigned cd_values[], size_t nbytes,
- size_t *buf_size, void **buf);
+H5_DLL herr_t H5Zunregister(H5Z_filter_t id);
+
+H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id);
+
-size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
- const unsigned cd_values[], size_t nbytes,
- size_t *buf_size, void **buf);
#ifdef __cplusplus
}
#endif
diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c
index 61dcc4f..ffd676f 100644
--- a/src/H5Zshuffle.c
+++ b/src/H5Zshuffle.c
@@ -9,73 +9,106 @@
#include "H5Eprivate.h"
#include "H5MMprivate.h"
#include "H5Zprivate.h"
-#include <stdio.h>
#ifdef H5_HAVE_FILTER_SHUFFLE
-size_t H5Z_filter_shuffle(unsigned flags,
- size_t cd_nelmts,
- const unsigned cd_values[],
- size_t nbytes,
- size_t *buf_size,void **buf) {
-
- size_t i;
- size_t j;
- size_t k;
- size_t ret_value = 0;
- size_t byte_pos;
- size_t bytesoftype;
- void* dest = NULL;
- char* _src;
- char* _des;
- char* _dest;
- size_t numofelements;
-
- bytesoftype=cd_values[0];
- numofelements=nbytes/bytesoftype;
- _src =(char*)(*buf);
+/* Interface initialization */
+#define PABLO_MASK H5Z_shuffle_mask
+#define INTERFACE_INIT NULL
+static int interface_initialize_g = 0;
- dest = malloc((size_t)nbytes);
- _dest =(char*)dest;
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_filter_shuffle
+ *
+ * Purpose: Implement an I/O filter which "de-interlaces" a block of data
+ * by putting all the bytes in a byte-position for each element
+ * together in the block. For example, for 4-byte elements stored
+ * as: 012301230123, shuffling will store them as: 000111222333
+ * Usually, the bytes in each byte position are more related to
+ * each other and putting them together will increase compression.
+ *
+ * Return: Success:
+ *
+ * Failure:
+ *
+ * Programmer: Kent Yang
+ * Wednesday, November 13, 2002
+ *
+ * Modifications:
+ * Quincey Koziol, November 13, 2002
+ * Cleaned up code.
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
+ const unsigned cd_values[], size_t nbytes,
+ size_t *buf_size, void **buf)
+{
+ void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */
+ unsigned char *_src; /* Alias for source buffer */
+ unsigned char *_dest; /* Alias for destination buffer */
+ unsigned bytesoftype; /* Number of bytes per element */
+ size_t numofelements; /* Number of elements in buffer */
+ size_t i,j; /* Local index variables */
+ size_t ret_value; /* Return value */
+
+ FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0);
+
+ /* Check arguments */
+ if (cd_nelmts!=1 || cd_values[0]==0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid shuffle parameters");
+
+ /* Get the number of bytes per element from the parameter block */
+ bytesoftype=cd_values[0];
+
+ /* Don't do anything for 1-byte elements */
+ if(bytesoftype>1) {
+ /* Compute the number of elements in buffer */
+ numofelements=nbytes/bytesoftype;
+
+ /* Allocate the destination buffer */
+ if (NULL==(dest = H5MM_malloc(nbytes)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer");
+
+ if(flags & H5Z_FLAG_REVERSE) {
+ /* Get the pointer to the source buffer */
+ _src =(unsigned char *)(*buf);
+
+ /* Input; unshuffle */
+ for(i=0; i<bytesoftype; i++) {
+ _dest=((unsigned char *)dest)+i;
+ for(j=0; j<numofelements; j++) {
+ *_dest=*_src++;
+ _dest+=bytesoftype;
+ } /* end for */
+ } /* end for */
+ } /* end if */
+ else {
+ /* Get the pointer to the destination buffer */
+ _dest =(unsigned char *)dest;
- j = 0;
- k = 0;
- if(flags & H5Z_FLAG_REVERSE) {
- for(i=0;i<nbytes;i++) {
- byte_pos = 1 +j *numofelements;
- if(byte_pos > nbytes) {
- k++;
- j=0;
- byte_pos=1;
- }
- *(_dest+i)=*((char*)(_src+byte_pos-1+k));
- j++;
- }
- free(*buf);
- *buf = dest;
- dest = NULL;
- *buf_size=nbytes;
- ret_value = nbytes;
- }
+ /* Output; shuffle */
+ for(i=0; i<bytesoftype; i++) {
+ _src=((unsigned char *)(*buf))+i;
+ for(j=0; j<numofelements; j++) {
+ *_dest++=*_src;
+ _src+=bytesoftype;
+ } /* end for */
+ } /* end for */
+ } /* end else */
- else {
- for (i=0;i<nbytes;i++){
- byte_pos = 1+j * bytesoftype;
- if(byte_pos >nbytes) {
- k++;
- j=0;
- byte_pos = 1;
- }
+ /* Set the buffer information to return */
+ H5MM_xfree(*buf);
+ *buf = dest;
+ *buf_size=nbytes;
+ } /* end else */
- *(_dest+i)=*(_src+byte_pos-1+k);
- j++;
- }
- free(*buf);
- *buf = dest;
- dest = NULL;
- *buf_size=nbytes;
- ret_value = nbytes;
- }
+ /* Set the return value */
+ ret_value = nbytes;
+done:
+ FUNC_LEAVE (ret_value);
}
#endif /*H5_HAVE_FILTER_SHUFFLE */
diff --git a/src/H5config.h.in b/src/H5config.h.in
index 7a45f6c..498225f 100644
--- a/src/H5config.h.in
+++ b/src/H5config.h.in
@@ -9,9 +9,6 @@
/* Define to 1 if you have the `BSDgettimeofday' function. */
#undef HAVE_BSDGETTIMEOFDAY
-/* Define if we have zlib compression support */
-#undef HAVE_COMPRESSION
-
/* Define to 1 if you have the `difftime' function. */
#undef HAVE_DIFFTIME
@@ -24,8 +21,8 @@
/* Define to 1 if you have the <features.h> header file. */
#undef HAVE_FEATURES_H
-/* Define if support for gzip filter is enabled */
-#undef HAVE_FILTER_GZIP
+/* Define if support for deflate filter is enabled */
+#undef HAVE_FILTER_DEFLATE
/* Define if support for shuffle filter is enabled */
#undef HAVE_FILTER_SHUFFLE