From fedd036348a6fa1f60bc1af3760ff34e2a47adaa Mon Sep 17 00:00:00 2001 From: "Robert E. McGrath" Date: Wed, 23 Feb 2005 12:52:03 -0500 Subject: [svn-r10067] Purpose: feature Description: h5repack support for scaleoffset compression Checking in early to help debug the filter. Solution: Added messages and command line to handle new scale offset filter. Note: TESTS ARE DISABLED FOR NOW. The filter is not complete, repack tests may fail due to know problems. PLEASE DO NOT MESS WITH THE SCALEOFFSET TESTS AT THIS TIME. They will be enabled when the filter is ready. Platforms tested: verbena,copper,shanti Misc. update: MANIFEST --- MANIFEST | 1 + tools/h5repack/h5repack.h | 2 + tools/h5repack/h5repack.sh.in | 31 ++++++++++- tools/h5repack/h5repack_copy.c | 3 ++ tools/h5repack/h5repack_filters.c | 14 ++++- tools/h5repack/h5repack_parse.c | 23 ++++++++ tools/h5repack/testh5repack_main.c | 60 +++++++++++++++++++++ tools/h5repack/testh5repack_make.c | 102 ++++++++++++++++++++++++++++++++++++ tools/testfiles/test_scaleoffset.h5 | Bin 0 -> 11016 bytes 9 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 tools/testfiles/test_scaleoffset.h5 diff --git a/MANIFEST b/MANIFEST index d01db53..6e058d5 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1545,6 +1545,7 @@ ./tools/testfiles/test_shuffle.h5 ./tools/testfiles/test_szip.h5 ./tools/testfiles/test_nbit.h5 +./tools/testfiles/test_scaleoffset.h5 ./tools/testfiles/h5repack_info.txt # jam utility and tests diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index a530b1b..434131c 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -277,6 +277,8 @@ int parse_number(char *str); #define FNAME11OUT "test_all.out.h5" #define FNAME12 "test_nbit.h5" #define FNAME12OUT "test_nbit.out.h5" +#define FNAME13 "test_scaleoffset.h5" +#define FNAME13OUT "test_scaleoffset.out.h5" int make_testfiles(void); diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 1ca28b7..bebfeab 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -19,6 +19,11 @@ USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@" USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@" USE_FILTER_NBIT="@USE_FILTER_NBIT@" +# +# Leave these tests off for now. Enable when filter is ready +: +#USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@" +USE_FILTER_SCALEOFFSET="no" H5REPACK=h5repack # The tool name @@ -307,9 +312,33 @@ else TOOLTEST $arg fi +# scaleoffset add +arg="test_scaleoffset.h5 -f dset_none:S+O=31" +if test $USE_FILTER_SCALEOFFSET != "yes" ; then + SKIP $arg +else + TOOLTEST $arg +fi + +# scaleoffset copy +arg="test_scaleoffset.h5" +if test $USE_FILTER_SCALEOFFSET != "yes" ; then + SKIP $arg +else + TOOLTEST $arg +fi + +# scaleoffset remove +arg="test_scaleoffset.h5 -f dset_scaleoffset:NONE" +if test $USE_FILTER_SCALEOFFSET != "yes" ; then + SKIP $arg +else + TOOLTEST $arg +fi + # remove all filters arg="test_all.h5 -f NONE" -if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" ; then +if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" -o $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else TOOLTEST $arg diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 3f558fc..05ea7fa 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -118,6 +118,9 @@ static void print_obj(hid_t dcpl_id, char *name) case H5Z_FILTER_NBIT: strcat(str,"NBIT "); break; + case H5Z_FILTER_SCALEOFFSET: + strcat(str,"SCALEOFFSET "); + break; } /* switch */ }/*i*/ diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index c9d6e20..76bfa34 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -288,6 +288,7 @@ int apply_filters(const char* name, /* object name from traverse list */ * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC * H5Z_FILTER_SZIP 4 , szip compression * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression *------------------------------------------------------------------------- */ @@ -370,7 +371,7 @@ int apply_filters(const char* name, /* object name from traverse list */ if (H5Pset_fletcher32(dcpl_id)<0) return -1; break; - /*------------------------------------------------------------------------- + /*----------- ------------------------------------------------------------- * H5Z_FILTER_NBIT , NBIT compression *------------------------------------------------------------------------- */ @@ -380,6 +381,17 @@ int apply_filters(const char* name, /* object name from traverse list */ if (H5Pset_nbit(dcpl_id)<0) return -1; break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET , scale+offset compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SCALEOFFSET: + aggression=obj.filter[i].cd_values[0]; + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_scaleoffset(dcpl_id,aggression)<0) + return -1; + break; } /* switch */ }/*i*/ diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index 9cd9b7d..1fd61ca 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -35,6 +35,7 @@ * SHUF, to apply the HDF5 shuffle filter * FLET, to apply the HDF5 checksum filter * NBIT, to apply the HDF5 NBIT filter (NBIT compression) + * S+O, to apply the HDF5 scale+offset filter (compression) * NONE, to remove the filter * * Examples: @@ -284,6 +285,19 @@ obj_list_t* parse_filter(const char *str, exit(1); } } +/*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"S+O")==0) + { + filt->filtn=H5Z_FILTER_SCALEOFFSET; + if (no_param) { /*no more parameters, S+O must have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Missing compression parameter in <%s>\n",str); + exit(1); + } + } else { if (obj_list) free(obj_list); printf("Input Error: Invalid filter type in <%s>\n",str); @@ -327,6 +341,13 @@ obj_list_t* parse_filter(const char *str, exit(1); } break; + case H5Z_FILTER_SCALEOFFSET: + if (filt->cd_values[0]<0 ){ + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression parameter in <%s>\n",str); + exit(1); + } + break; }; return obj_list; @@ -357,6 +378,8 @@ const char* get_sfilter(H5Z_filter_t filtn) return "FLETCHER32"; else if (filtn==H5Z_FILTER_NBIT) return "NBIT"; + else if (filtn==H5Z_FILTER_SCALEOFFSET) + return "S+O"; else { printf("Input error in filter type\n"); exit(1); diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 6c09dce..bc80c5a 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -1012,6 +1012,66 @@ if (szip_can_encode) { #else SKIPPED(); #endif + TESTING(" copy of scaleoffset filter"); + +#ifdef H5_HAVE_FILTER_SCALEOFFSET + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" removing scaleoffset filter"); + +#ifdef H5_HAVE_FILTER_SCALEOFFSET + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" adding scaleoffset filter"); + +#ifdef H5_HAVE_FILTER_SCALEOFFSET + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_none:S+O=31",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif /*------------------------------------------------------------------------- * file with all filters diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c index 4245bec..9ae625d 100644 --- a/tools/h5repack/testh5repack_make.c +++ b/tools/h5repack/testh5repack_make.c @@ -34,6 +34,7 @@ int make_deflate(hid_t loc_id); int make_shuffle(hid_t loc_id); int make_fletcher32(hid_t loc_id); int make_nbit(hid_t loc_id); +int make_scaleoffset(hid_t loc_id); int make_all(hid_t loc_id); int make_fill(hid_t loc_id); @@ -168,6 +169,15 @@ int make_testfiles(void) goto out; /*------------------------------------------------------------------------- + * create a file with the scaleoffset filter + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME13,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_scaleoffset(loc_id)<0) + goto out; + +/*------------------------------------------------------------------------- * create a file with the all filters *------------------------------------------------------------------------- */ @@ -735,6 +745,98 @@ out: /*------------------------------------------------------------------------- + * Function: make_scaleoffset + * + * Purpose: make a dataset with the scaleoffset filter + * + *------------------------------------------------------------------------- + */ +int make_scaleoffset(hid_t loc_id) +{ + hid_t dcpl; /* dataset creation property list */ + hid_t sid; /* dataspace ID */ + hid_t dtid; + hid_t dsid; + hsize_t dims[RANK]={DIM1,DIM2}; + hsize_t chunk_dims[RANK]={CDIM1,CDIM2}; + int buf[DIM1][DIM2]; + int i, j, n; + + for (i=n=0; i