summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobert E. McGrath <mcgrath@ncsa.uiuc.edu>2005-02-23 17:52:03 (GMT)
committerRobert E. McGrath <mcgrath@ncsa.uiuc.edu>2005-02-23 17:52:03 (GMT)
commitfedd036348a6fa1f60bc1af3760ff34e2a47adaa (patch)
tree646943e0abeb7d66f44431cb07a9c2d6f6059e6b
parent0b83fea12571ae1f05e6465d4afc09948486e17f (diff)
downloadhdf5-fedd036348a6fa1f60bc1af3760ff34e2a47adaa.zip
hdf5-fedd036348a6fa1f60bc1af3760ff34e2a47adaa.tar.gz
hdf5-fedd036348a6fa1f60bc1af3760ff34e2a47adaa.tar.bz2
[svn-r10067] Purpose:
feature Description: h5repack support for scaleoffset compression Checking in early to help debug the filter. Solution: Added messages and command line to handle new scale offset filter. Note: TESTS ARE DISABLED FOR NOW. The filter is not complete, repack tests may fail due to know problems. PLEASE DO NOT MESS WITH THE SCALEOFFSET TESTS AT THIS TIME. They will be enabled when the filter is ready. Platforms tested: verbena,copper,shanti Misc. update: MANIFEST
-rw-r--r--MANIFEST1
-rw-r--r--tools/h5repack/h5repack.h2
-rwxr-xr-xtools/h5repack/h5repack.sh.in31
-rw-r--r--tools/h5repack/h5repack_copy.c3
-rw-r--r--tools/h5repack/h5repack_filters.c14
-rw-r--r--tools/h5repack/h5repack_parse.c23
-rw-r--r--tools/h5repack/testh5repack_main.c60
-rw-r--r--tools/h5repack/testh5repack_make.c102
-rw-r--r--tools/testfiles/test_scaleoffset.h5bin0 -> 11016 bytes
9 files changed, 234 insertions, 2 deletions
diff --git a/MANIFEST b/MANIFEST
index d01db53..6e058d5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1545,6 +1545,7 @@
./tools/testfiles/test_shuffle.h5
./tools/testfiles/test_szip.h5
./tools/testfiles/test_nbit.h5
+./tools/testfiles/test_scaleoffset.h5
./tools/testfiles/h5repack_info.txt
# jam utility and tests
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index a530b1b..434131c 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -277,6 +277,8 @@ int parse_number(char *str);
#define FNAME11OUT "test_all.out.h5"
#define FNAME12 "test_nbit.h5"
#define FNAME12OUT "test_nbit.out.h5"
+#define FNAME13 "test_scaleoffset.h5"
+#define FNAME13OUT "test_scaleoffset.out.h5"
int make_testfiles(void);
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 1ca28b7..bebfeab 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -19,6 +19,11 @@ USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
USE_FILTER_NBIT="@USE_FILTER_NBIT@"
+#
+# Leave these tests off for now. Enable when filter is ready
+:
+#USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@"
+USE_FILTER_SCALEOFFSET="no"
H5REPACK=h5repack # The tool name
@@ -307,9 +312,33 @@ else
TOOLTEST $arg
fi
+# scaleoffset add
+arg="test_scaleoffset.h5 -f dset_none:S+O=31"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
+# scaleoffset copy
+arg="test_scaleoffset.h5"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
+# scaleoffset remove
+arg="test_scaleoffset.h5 -f dset_scaleoffset:NONE"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
# remove all filters
arg="test_all.h5 -f NONE"
-if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" ; then
+if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" -o $USE_FILTER_SCALEOFFSET != "yes" ; then
SKIP $arg
else
TOOLTEST $arg
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 3f558fc..05ea7fa 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -118,6 +118,9 @@ static void print_obj(hid_t dcpl_id, char *name)
case H5Z_FILTER_NBIT:
strcat(str,"NBIT ");
break;
+ case H5Z_FILTER_SCALEOFFSET:
+ strcat(str,"SCALEOFFSET ");
+ break;
} /* switch */
}/*i*/
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index c9d6e20..76bfa34 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -288,6 +288,7 @@ int apply_filters(const char* name, /* object name from traverse list */
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
+ * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*-------------------------------------------------------------------------
*/
@@ -370,7 +371,7 @@ int apply_filters(const char* name, /* object name from traverse list */
if (H5Pset_fletcher32(dcpl_id)<0)
return -1;
break;
- /*-------------------------------------------------------------------------
+ /*----------- -------------------------------------------------------------
* H5Z_FILTER_NBIT , NBIT compression
*-------------------------------------------------------------------------
*/
@@ -380,6 +381,17 @@ int apply_filters(const char* name, /* object name from traverse list */
if (H5Pset_nbit(dcpl_id)<0)
return -1;
break;
+ /*----------- -------------------------------------------------------------
+ * H5Z_FILTER_SCALEOFFSET , scale+offset compression
+ *-------------------------------------------------------------------------
+ */
+ case H5Z_FILTER_SCALEOFFSET:
+ aggression=obj.filter[i].cd_values[0];
+ if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
+ return -1;
+ if (H5Pset_scaleoffset(dcpl_id,aggression)<0)
+ return -1;
+ break;
} /* switch */
}/*i*/
diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c
index 9cd9b7d..1fd61ca 100644
--- a/tools/h5repack/h5repack_parse.c
+++ b/tools/h5repack/h5repack_parse.c
@@ -35,6 +35,7 @@
* SHUF, to apply the HDF5 shuffle filter
* FLET, to apply the HDF5 checksum filter
* NBIT, to apply the HDF5 NBIT filter (NBIT compression)
+ * S+O, to apply the HDF5 scale+offset filter (compression)
* NONE, to remove the filter
*
* Examples:
@@ -284,6 +285,19 @@ obj_list_t* parse_filter(const char *str,
exit(1);
}
}
+/*-------------------------------------------------------------------------
+ * H5Z_FILTER_SCALEOFFSET
+ *-------------------------------------------------------------------------
+ */
+ else if (strcmp(scomp,"S+O")==0)
+ {
+ filt->filtn=H5Z_FILTER_SCALEOFFSET;
+ if (no_param) { /*no more parameters, S+O must have parameter */
+ if (obj_list) free(obj_list);
+ printf("Input Error: Missing compression parameter in <%s>\n",str);
+ exit(1);
+ }
+ }
else {
if (obj_list) free(obj_list);
printf("Input Error: Invalid filter type in <%s>\n",str);
@@ -327,6 +341,13 @@ obj_list_t* parse_filter(const char *str,
exit(1);
}
break;
+ case H5Z_FILTER_SCALEOFFSET:
+ if (filt->cd_values[0]<0 ){
+ if (obj_list) free(obj_list);
+ printf("Input Error: Invalid compression parameter in <%s>\n",str);
+ exit(1);
+ }
+ break;
};
return obj_list;
@@ -357,6 +378,8 @@ const char* get_sfilter(H5Z_filter_t filtn)
return "FLETCHER32";
else if (filtn==H5Z_FILTER_NBIT)
return "NBIT";
+ else if (filtn==H5Z_FILTER_SCALEOFFSET)
+ return "S+O";
else {
printf("Input error in filter type\n");
exit(1);
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 6c09dce..bc80c5a 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -1012,6 +1012,66 @@ if (szip_can_encode) {
#else
SKIPPED();
#endif
+ TESTING(" copy of scaleoffset filter");
+
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+ TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+ TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+ TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+ TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+ TEST_ERROR;
+
+ PASSED();
+#else
+ SKIPPED();
+#endif
+
+ TESTING(" removing scaleoffset filter");
+
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+ TEST_ERROR;
+ if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options)<0)
+ TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+ TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+ TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+ TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+ TEST_ERROR;
+
+ PASSED();
+#else
+ SKIPPED();
+#endif
+
+ TESTING(" adding scaleoffset filter");
+
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+ TEST_ERROR;
+ if (h5repack_addfilter("dset_none:S+O=31",&pack_options)<0)
+ TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+ TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+ TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+ TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+ TEST_ERROR;
+
+ PASSED();
+#else
+ SKIPPED();
+#endif
/*-------------------------------------------------------------------------
* file with all filters
diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c
index 4245bec..9ae625d 100644
--- a/tools/h5repack/testh5repack_make.c
+++ b/tools/h5repack/testh5repack_make.c
@@ -34,6 +34,7 @@ int make_deflate(hid_t loc_id);
int make_shuffle(hid_t loc_id);
int make_fletcher32(hid_t loc_id);
int make_nbit(hid_t loc_id);
+int make_scaleoffset(hid_t loc_id);
int make_all(hid_t loc_id);
int make_fill(hid_t loc_id);
@@ -168,6 +169,15 @@ int make_testfiles(void)
goto out;
/*-------------------------------------------------------------------------
+ * create a file with the scaleoffset filter
+ *-------------------------------------------------------------------------
+ */
+ if((loc_id = H5Fcreate(FNAME13,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
+ return -1;
+ if (make_scaleoffset(loc_id)<0)
+ goto out;
+
+/*-------------------------------------------------------------------------
* create a file with the all filters
*-------------------------------------------------------------------------
*/
@@ -735,6 +745,98 @@ out:
/*-------------------------------------------------------------------------
+ * Function: make_scaleoffset
+ *
+ * Purpose: make a dataset with the scaleoffset filter
+ *
+ *-------------------------------------------------------------------------
+ */
+int make_scaleoffset(hid_t loc_id)
+{
+ hid_t dcpl; /* dataset creation property list */
+ hid_t sid; /* dataspace ID */
+ hid_t dtid;
+ hid_t dsid;
+ hsize_t dims[RANK]={DIM1,DIM2};
+ hsize_t chunk_dims[RANK]={CDIM1,CDIM2};
+ int buf[DIM1][DIM2];
+ int i, j, n;
+
+ for (i=n=0; i<DIM1; i++){
+ for (j=0; j<DIM2; j++){
+ buf[i][j]=n++;
+ }
+ }
+ /* create a space */
+ if((sid = H5Screate_simple(RANK, dims, NULL))<0)
+ return -1;
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0)
+ goto out;
+ /* set up chunk */
+ if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
+ goto out;
+
+ dtid = H5Tcopy(H5T_NATIVE_INT);
+
+#if defined (H5_HAVE_FILTER_NBIT)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ if (H5Pset_scaleoffset(dcpl,31)<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ if((dsid = H5Dcreate (loc_id,"dset_scaleoffset",dtid,sid,dcpl))<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ H5Dclose(dsid);
+ if((dsid = H5Dcreate (loc_id,"dset_none",dtid,sid,H5P_DEFAULT))<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+ H5Tclose(dtid);
+ goto out;
+ }
+ H5Tclose(dtid);
+ H5Dclose(dsid);
+#endif
+
+/*-------------------------------------------------------------------------
+ * close space and dcpl
+ *-------------------------------------------------------------------------
+ */
+ if(H5Sclose(sid)<0)
+ goto out;
+ if(H5Pclose(dcpl)<0)
+ goto out;
+
+ return 0;
+
+out:
+ H5E_BEGIN_TRY {
+ H5Pclose(dcpl);
+ H5Sclose(sid);
+ } H5E_END_TRY;
+ return -1;
+}
+
+
+/*-------------------------------------------------------------------------
* Function: make_all
*
* Purpose: make a file with all filters
diff --git a/tools/testfiles/test_scaleoffset.h5 b/tools/testfiles/test_scaleoffset.h5
new file mode 100644
index 0000000..4e78d83
--- /dev/null
+++ b/tools/testfiles/test_scaleoffset.h5
Binary files differ