From 55de236da8c16b94f6c8bec17cadc3ca69daed90 Mon Sep 17 00:00:00 2001 From: MuQun Yang Date: Wed, 13 Nov 2002 11:35:11 -0500 Subject: [svn-r6087] Purpose: Adding internal shuffle filter Description: With the combination of shuffling filter with general compression algorithm, the compression ratio may be improved without adding much encoding and decoding time for many real NASA datasets(especially floating data) and other application datasets(See techNotes). Solution: SHuffle the bytes within the data to utilize the locality. Platforms tested: arabica , eirene, modi4 Misc. update: Update MANIFEST if you add or remove any file. Update release_docs/RELEASE for bug fixes, new features, etc. Update applicable document files too. --- src/H5Pdcpl.c | 55 ++++++++++++++++++++++++++++++++++++++ src/H5Ppublic.h | 1 + src/H5Zpublic.h | 4 +++ src/H5Zshuffle.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+) create mode 100644 src/H5Zshuffle.c diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 7e995ef..d7f8809 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -751,6 +751,61 @@ done: FUNC_LEAVE(ret_value); } + +/*------------------------------------------------------------------------- + * Function: H5Pset_shuffle + * + * Purpose: Sets the shuffling method for a permanent + * filter + * to H5Z_FILTER_SHUFFLE + * and bytes of the datatype of the array to be shuffled + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + * Raymond Lu + * Tuesday, October 2, 2001 + * Changed the way to check parameter and set property for + * generic property list. + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_shuffle(hid_t plist_id, unsigned bytes_of_type) +{ + H5O_pline_t pline; + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + + FUNC_ENTER_API(H5Pset_shuffle, FAIL); + H5TRACE2("e","iIu",plist_id,bytes_of_type); + + /* Check arguments */ + if(TRUE != H5P_isa_class(plist_id, H5P_DATASET_CREATE)) + HGOTO_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list"); + if (bytes_of_type == 0) + HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "number of bytes of datatype cannot be 0"); + + /* Get the plist structure */ + if(NULL == (plist = H5I_object(plist_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Add the filter */ + if(H5P_get(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + if (H5Z_append(&pline, H5Z_FILTER_SHUFFLE, H5Z_FLAG_OPTIONAL, 1, &bytes_of_type)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to shuffle the data"); + if(H5P_set(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to set pipeline"); + +done: + FUNC_LEAVE(ret_value); +} + /*------------------------------------------------------------------------- * Function: H5Pset_fill_value diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index b0b332e..e1a6cf1 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -210,6 +210,7 @@ H5_DLL H5Z_filter_t H5Pget_filter(hid_t plist_id, int filter, unsigned cd_values[]/*out*/, size_t namelen, char name[]); H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned aggression); +H5_DLL herr_t H5Pset_shuffle(hid_t plist_id, unsigned bytespertype); #ifdef H5_WANT_H5_V1_4_COMPAT H5_DLL herr_t H5Pset_cache(hid_t plist_id, int mdc_nelmts, int rdcc_nelmts, size_t rdcc_nbytes, double rdcc_w0); diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 466fbb6..46f74a3 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -19,6 +19,7 @@ typedef int H5Z_filter_t; #define H5Z_FILTER_ERROR (-1) /*no filter */ #define H5Z_FILTER_NONE 0 /*reserved indefinitely */ #define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ +#define H5Z_FILTER_SHUFFLE 2 /* shuffle the data */ #define H5Z_FILTER_MAX 65535 /*maximum filter id */ /* Flags for filter definition */ @@ -59,6 +60,9 @@ size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); +size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); #ifdef __cplusplus } #endif diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c new file mode 100644 index 0000000..61dcc4f --- /dev/null +++ b/src/H5Zshuffle.c @@ -0,0 +1,81 @@ +/* + * Copyright © 1999-2001 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke + * Friday, August 27, 1999 + */ +#include "H5private.h" +#include "H5Eprivate.h" +#include "H5MMprivate.h" +#include "H5Zprivate.h" +#include + +#ifdef H5_HAVE_FILTER_SHUFFLE + +size_t H5Z_filter_shuffle(unsigned flags, + size_t cd_nelmts, + const unsigned cd_values[], + size_t nbytes, + size_t *buf_size,void **buf) { + + size_t i; + size_t j; + size_t k; + size_t ret_value = 0; + size_t byte_pos; + size_t bytesoftype; + void* dest = NULL; + char* _src; + char* _des; + char* _dest; + size_t numofelements; + + bytesoftype=cd_values[0]; + numofelements=nbytes/bytesoftype; + _src =(char*)(*buf); + + dest = malloc((size_t)nbytes); + _dest =(char*)dest; + + j = 0; + k = 0; + if(flags & H5Z_FLAG_REVERSE) { + for(i=0;i nbytes) { + k++; + j=0; + byte_pos=1; + } + *(_dest+i)=*((char*)(_src+byte_pos-1+k)); + j++; + } + free(*buf); + *buf = dest; + dest = NULL; + *buf_size=nbytes; + ret_value = nbytes; + } + + else { + for (i=0;inbytes) { + k++; + j=0; + byte_pos = 1; + } + + *(_dest+i)=*(_src+byte_pos-1+k); + j++; + } + free(*buf); + *buf = dest; + dest = NULL; + *buf_size=nbytes; + ret_value = nbytes; + } + +} +#endif /*H5_HAVE_FILTER_SHUFFLE */ -- cgit v0.12