From ac48a23e2b72b03d60d4fdf10957ed31beb5b70d Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Wed, 20 Nov 2002 08:15:18 -0500 Subject: [svn-r6114] Purpose: Code Cleanup & New Feature Description: H5config.h.in: Removed H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags. Added H5_HAVE_FILTER_DEFLATE flag. H5Z.c: H5Zprivate.h: H5Zpublic.h: Switched from using H5_HAVE_COMPRESSION flag in favor of H5_HAVE_FILTER_DEFLATE. Added H5Zunregister & H5Zfilter_avail API functions. Changed a numeric constant (256) to a symbolic constant (H5Z_FILTER_RESERVED). Automatically add the shuffling filter to the list of available filters (when it is enabled). Moved prototypes for H5Z_filter_deflate & H5Z_filter_shuffle from the public header into the private header. H5Zdeflate.c: Switched from using H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags in favor of H5_HAVE_FILTER_DEFLATE. Cleaned up formatting & error reporting a bit. H5Zshuffle.c: Rewrote shuffling algorithm to be more efficient. Added error checking & reporting. Added standard Pablo information. Added standard function header comment. Added FUNC_ENTER & FUNC_LEAVE macros. Platforms tested: Tested h5committest {arabica (fortran), eirene (fortran, C++) modi4 (parallel, fortran)} FreeBSD 4.7 (sleipnir) --- src/H5Z.c | 133 +++++++++++++++++++++++++++++++++++++++++++++-- src/H5Zdeflate.c | 55 ++++++-------------- src/H5Zprivate.h | 9 ++++ src/H5Zpublic.h | 11 ++-- src/H5Zshuffle.c | 153 +++++++++++++++++++++++++++++++++--------------------- src/H5config.h.in | 7 +-- 6 files changed, 253 insertions(+), 115 deletions(-) diff --git a/src/H5Z.c b/src/H5Z.c index 4016481..9c240c3 100644 --- a/src/H5Z.c +++ b/src/H5Z.c @@ -19,10 +19,12 @@ static int interface_initialize_g = 0; static herr_t H5Z_init_interface (void); +/* Local variables */ static size_t H5Z_table_alloc_g = 0; static size_t H5Z_table_used_g = 0; static H5Z_class_t *H5Z_table_g = NULL; +/* Local functions */ /*------------------------------------------------------------------------- @@ -44,9 +46,12 @@ H5Z_init_interface (void) { FUNC_ENTER_NOINIT(H5Z_init_interface); -#ifdef H5_HAVE_FILTER_GZIP +#ifdef H5_HAVE_FILTER_DEFLATE H5Z_register (H5Z_FILTER_DEFLATE, "deflate", H5Z_filter_deflate); -#endif /* H5_HAVE_FILTER_GZIP */ +#endif /* H5_HAVE_FILTER_DEFLATE */ +#ifdef H5_HAVE_FILTER_SHUFFLE + H5Z_register (H5Z_FILTER_SHUFFLE, "shuffle", H5Z_filter_shuffle); +#endif /* H5_HAVE_FILTER_SHUFFLE */ FUNC_LEAVE (SUCCEED); } @@ -162,7 +167,7 @@ H5Zregister(H5Z_filter_t id, const char *comment, H5Z_func_t func) /* Check args */ if (id<0 || id>H5Z_FILTER_MAX) HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number"); - if (id<256) + if (idH5Z_FILTER_MAX) + HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number"); + if (id=0 && id<=H5Z_FILTER_MAX); + + /* Is the filter already registered? */ + for (i=0; i=H5Z_table_used_g) + HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter is not registered"); + + /* Remove filter from table */ + /* Don't worry about shrinking table size (for now) */ + HDmemmove(&H5Z_table_g[i],&H5Z_table_g[i+1],sizeof(H5Z_class_t)*((H5Z_table_used_g-1)-i)); + H5Z_table_used_g--; + +done: + FUNC_LEAVE (ret_value); +} /* end H5Z_unregister() */ + + +/*------------------------------------------------------------------------- + * Function: H5Zfilter_avail + * + * Purpose: Check if a filter is available + * + * Return: Non-negative (TRUE/FALSE) on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, November 14, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +htri_t +H5Zfilter_avail(H5Z_filter_t id) +{ + size_t i; /* Local index variable */ + htri_t ret_value=FALSE; /* Return value */ + + FUNC_ENTER_API(H5Zfilter_avail, FAIL); + H5TRACE1("b","Zf",id); + + /* Check args */ + if(id<0 || id>H5Z_FILTER_MAX) + HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number"); + + /* Is the filter already registered? */ + for(i=0; i9) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, - "invalid deflate aggression level"); - } + if (cd_nelmts!=1 || cd_values[0]>9) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level"); -#if defined(H5_HAVE_COMPRESSION) aggression = cd_values[0]; if (flags & H5Z_FLAG_REVERSE) { /* Input; uncompress */ z_stream z_strm; size_t nalloc = *buf_size; - if (NULL==(outbuf = H5MM_malloc(nalloc))) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, - "memory allocation failed for deflate uncompression"); - } + if (NULL==(outbuf = H5MM_malloc(nalloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression"); HDmemset(&z_strm, 0, sizeof(z_strm)); z_strm.next_in = *buf; H5_ASSIGN_OVERFLOW(z_strm.avail_in,nbytes,size_t,uInt); z_strm.next_out = outbuf; H5_ASSIGN_OVERFLOW(z_strm.avail_out,nalloc,size_t,uInt); - if (Z_OK!=inflateInit(&z_strm)) { + if (Z_OK!=inflateInit(&z_strm)) HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed"); - } while (1) { status = inflate(&z_strm, Z_SYNC_FLUSH); - if (Z_STREAM_END==status) break; /*done*/ + if (Z_STREAM_END==status) + break; /*done*/ if (Z_OK!=status) { inflateEnd(&z_strm); HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed"); @@ -97,9 +82,7 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts, nalloc *= 2; if (NULL==(outbuf = H5MM_realloc(outbuf, nalloc))) { inflateEnd(&z_strm); - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, - "memory allocation failed for deflate " - "uncompression"); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression"); } z_strm.next_out = (unsigned char*)outbuf + z_strm.total_out; z_strm.avail_out = (uInt)(nalloc - z_strm.total_out); @@ -112,7 +95,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts, *buf_size = nalloc; ret_value = z_strm.total_out; inflateEnd(&z_strm); - } else { /* * Output; compress but fail if the result would be larger than the @@ -124,12 +106,9 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts, uLongf z_dst_nbytes = (uLongf)nbytes; uLong z_src_nbytes = (uLong)nbytes; - if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, - "unable to allocate deflate destination buffer"); - } - status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, - aggression); + if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate deflate destination buffer"); + status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); if (Z_BUF_ERROR==status) { HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow"); } else if (Z_MEM_ERROR==status) { @@ -144,10 +123,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts, ret_value = z_dst_nbytes; } } -#else - HGOTO_ERROR (H5E_PLINE, H5E_UNSUPPORTED, 0, - "hdf5 was not compiled with zlib-1.0.2 or better"); -#endif done: if(outbuf) @@ -155,4 +130,4 @@ done: FUNC_LEAVE (ret_value); } -#endif /* H5_HAVE_FILTER_GZIP */ +#endif /* H5_HAVE_FILTER_DEFLATE */ diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 0a20b13..388f204 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -33,6 +33,7 @@ struct H5O_pline_t; /*forward decl*/ H5_DLL herr_t H5Z_register(H5Z_filter_t id, const char *comment, H5Z_func_t filter); +H5_DLL herr_t H5Z_unregister (H5Z_filter_t id); H5_DLL herr_t H5Z_append(struct H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, size_t cd_nelmts, const unsigned int cd_values[]); @@ -43,4 +44,12 @@ H5_DLL herr_t H5Z_pipeline(H5F_t *f, const struct H5O_pline_t *pline, H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id); +/* Filter routines */ +size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); #endif diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 46f74a3..0b9a0bf 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -20,6 +20,7 @@ typedef int H5Z_filter_t; #define H5Z_FILTER_NONE 0 /*reserved indefinitely */ #define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ #define H5Z_FILTER_SHUFFLE 2 /* shuffle the data */ +#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved */ #define H5Z_FILTER_MAX 65535 /*maximum filter id */ /* Flags for filter definition */ @@ -56,13 +57,11 @@ extern "C" { H5_DLL herr_t H5Zregister(H5Z_filter_t id, const char *comment, H5Z_func_t filter); -size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); +H5_DLL herr_t H5Zunregister(H5Z_filter_t id); + +H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id); + -size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); #ifdef __cplusplus } #endif diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index 61dcc4f..ffd676f 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -9,73 +9,106 @@ #include "H5Eprivate.h" #include "H5MMprivate.h" #include "H5Zprivate.h" -#include #ifdef H5_HAVE_FILTER_SHUFFLE -size_t H5Z_filter_shuffle(unsigned flags, - size_t cd_nelmts, - const unsigned cd_values[], - size_t nbytes, - size_t *buf_size,void **buf) { - - size_t i; - size_t j; - size_t k; - size_t ret_value = 0; - size_t byte_pos; - size_t bytesoftype; - void* dest = NULL; - char* _src; - char* _des; - char* _dest; - size_t numofelements; - - bytesoftype=cd_values[0]; - numofelements=nbytes/bytesoftype; - _src =(char*)(*buf); +/* Interface initialization */ +#define PABLO_MASK H5Z_shuffle_mask +#define INTERFACE_INIT NULL +static int interface_initialize_g = 0; - dest = malloc((size_t)nbytes); - _dest =(char*)dest; + +/*------------------------------------------------------------------------- + * Function: H5Z_filter_shuffle + * + * Purpose: Implement an I/O filter which "de-interlaces" a block of data + * by putting all the bytes in a byte-position for each element + * together in the block. For example, for 4-byte elements stored + * as: 012301230123, shuffling will store them as: 000111222333 + * Usually, the bytes in each byte position are more related to + * each other and putting them together will increase compression. + * + * Return: Success: + * + * Failure: + * + * Programmer: Kent Yang + * Wednesday, November 13, 2002 + * + * Modifications: + * Quincey Koziol, November 13, 2002 + * Cleaned up code. + * + *------------------------------------------------------------------------- + */ +size_t +H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */ + unsigned char *_src; /* Alias for source buffer */ + unsigned char *_dest; /* Alias for destination buffer */ + unsigned bytesoftype; /* Number of bytes per element */ + size_t numofelements; /* Number of elements in buffer */ + size_t i,j; /* Local index variables */ + size_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0); + + /* Check arguments */ + if (cd_nelmts!=1 || cd_values[0]==0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid shuffle parameters"); + + /* Get the number of bytes per element from the parameter block */ + bytesoftype=cd_values[0]; + + /* Don't do anything for 1-byte elements */ + if(bytesoftype>1) { + /* Compute the number of elements in buffer */ + numofelements=nbytes/bytesoftype; + + /* Allocate the destination buffer */ + if (NULL==(dest = H5MM_malloc(nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer"); + + if(flags & H5Z_FLAG_REVERSE) { + /* Get the pointer to the source buffer */ + _src =(unsigned char *)(*buf); + + /* Input; unshuffle */ + for(i=0; i nbytes) { - k++; - j=0; - byte_pos=1; - } - *(_dest+i)=*((char*)(_src+byte_pos-1+k)); - j++; - } - free(*buf); - *buf = dest; - dest = NULL; - *buf_size=nbytes; - ret_value = nbytes; - } + /* Output; shuffle */ + for(i=0; inbytes) { - k++; - j=0; - byte_pos = 1; - } + /* Set the buffer information to return */ + H5MM_xfree(*buf); + *buf = dest; + *buf_size=nbytes; + } /* end else */ - *(_dest+i)=*(_src+byte_pos-1+k); - j++; - } - free(*buf); - *buf = dest; - dest = NULL; - *buf_size=nbytes; - ret_value = nbytes; - } + /* Set the return value */ + ret_value = nbytes; +done: + FUNC_LEAVE (ret_value); } #endif /*H5_HAVE_FILTER_SHUFFLE */ diff --git a/src/H5config.h.in b/src/H5config.h.in index 7a45f6c..498225f 100644 --- a/src/H5config.h.in +++ b/src/H5config.h.in @@ -9,9 +9,6 @@ /* Define to 1 if you have the `BSDgettimeofday' function. */ #undef HAVE_BSDGETTIMEOFDAY -/* Define if we have zlib compression support */ -#undef HAVE_COMPRESSION - /* Define to 1 if you have the `difftime' function. */ #undef HAVE_DIFFTIME @@ -24,8 +21,8 @@ /* Define to 1 if you have the header file. */ #undef HAVE_FEATURES_H -/* Define if support for gzip filter is enabled */ -#undef HAVE_FILTER_GZIP +/* Define if support for deflate filter is enabled */ +#undef HAVE_FILTER_DEFLATE /* Define if support for shuffle filter is enabled */ #undef HAVE_FILTER_SHUFFLE -- cgit v0.12