[svn-r10067] Purpose:

feature

Description:
h5repack support for scaleoffset compression

Checking in early to help debug the filter.

Solution:
Added messages and command line to handle new scale offset filter.

Note:  TESTS ARE DISABLED FOR NOW.  The filter is not
complete, repack tests may fail due to know problems.

PLEASE DO NOT MESS WITH THE SCALEOFFSET TESTS AT THIS TIME.

They will be enabled when the filter is ready.

Platforms tested:
verbena,copper,shanti

Misc. update:
MANIFEST

9 files changed, 234 insertions, 2 deletions

diff --git a/MANIFEST b/MANIFEST
index d01db53..6e058d5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1545,6 +1545,7 @@
 ./tools/testfiles/test_shuffle.h5
 ./tools/testfiles/test_szip.h5
 ./tools/testfiles/test_nbit.h5
+./tools/testfiles/test_scaleoffset.h5
 ./tools/testfiles/h5repack_info.txt
 
 # jam utility and tests
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index a530b1b..434131c 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -277,6 +277,8 @@ int         parse_number(char *str);
 #define FNAME11OUT    "test_all.out.h5"
 #define FNAME12    "test_nbit.h5"
 #define FNAME12OUT    "test_nbit.out.h5"
+#define FNAME13    "test_scaleoffset.h5"
+#define FNAME13OUT    "test_scaleoffset.out.h5"
 
 int make_testfiles(void);
 
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 1ca28b7..bebfeab 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -19,6 +19,11 @@ USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
 USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
 USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
 USE_FILTER_NBIT="@USE_FILTER_NBIT@"
+#
+#  Leave these tests off for now. Enable when filter is ready
+:
+#USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@"
+USE_FILTER_SCALEOFFSET="no"
 
 
 H5REPACK=h5repack               # The tool name
@@ -307,9 +312,33 @@ else
  TOOLTEST $arg
 fi
 
+# scaleoffset add
+arg="test_scaleoffset.h5 -f dset_none:S+O=31"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
+# scaleoffset copy
+arg="test_scaleoffset.h5"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
+# scaleoffset remove
+arg="test_scaleoffset.h5 -f dset_scaleoffset:NONE"
+if test $USE_FILTER_SCALEOFFSET != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi        
+
 # remove all  filters
 arg="test_all.h5 -f NONE"
-if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes"  -o $USE_FILTER_NBIT != "yes"  ; then
+if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes"  -o $USE_FILTER_NBIT != "yes"  -o $USE_FILTER_SCALEOFFSET != "yes"  ; then
  SKIP $arg
 else
  TOOLTEST $arg
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 3f558fc..05ea7fa 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -118,6 +118,9 @@ static void print_obj(hid_t dcpl_id, char *name)
   case H5Z_FILTER_NBIT:
    strcat(str,"NBIT ");
    break;
+  case H5Z_FILTER_SCALEOFFSET:
+   strcat(str,"SCALEOFFSET ");
+   break;
   } /* switch */
  }/*i*/
 
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index c9d6e20..76bfa34 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -288,6 +288,7 @@ int apply_filters(const char* name,    /* object name from traverse list */
  * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
  * H5Z_FILTER_SZIP       4 , szip compression 
  * H5Z_FILTER_NBIT       5 , nbit compression 
+ * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression 
  *-------------------------------------------------------------------------
  */
 	
@@ -370,7 +371,7 @@ int apply_filters(const char* name,    /* object name from traverse list */
 				if (H5Pset_fletcher32(dcpl_id)<0) 
 					return -1;
 				break;
-				/*-------------------------------------------------------------------------
+				/*----------- -------------------------------------------------------------
 				* H5Z_FILTER_NBIT , NBIT compression
 				*-------------------------------------------------------------------------
 				*/
@@ -380,6 +381,17 @@ int apply_filters(const char* name,    /* object name from traverse list */
 				if (H5Pset_nbit(dcpl_id)<0) 
 					return -1;
 				break;
+				/*----------- -------------------------------------------------------------
+				* H5Z_FILTER_SCALEOFFSET , scale+offset compression
+				*-------------------------------------------------------------------------
+				*/
+			case H5Z_FILTER_SCALEOFFSET:
+				aggression=obj.filter[i].cd_values[0];
+				if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
+					return -1;
+				if (H5Pset_scaleoffset(dcpl_id,aggression)<0) 
+					return -1;
+				break;
 			} /* switch */
 		}/*i*/
 		
diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c
index 9cd9b7d..1fd61ca 100644
--- a/tools/h5repack/h5repack_parse.c
+++ b/tools/h5repack/h5repack_parse.c
@@ -35,6 +35,7 @@
  *  SHUF, to apply the HDF5 shuffle filter
  *  FLET, to apply the HDF5 checksum filter
  *  NBIT, to apply the HDF5 NBIT filter (NBIT compression)
+ *  S+O, to apply the HDF5 scale+offset filter (compression)
  *  NONE, to remove the filter 
  *
  * Examples: 
@@ -284,6 +285,19 @@ obj_list_t* parse_filter(const char *str,
      exit(1);
     }
    }
+/*-------------------------------------------------------------------------
+ * H5Z_FILTER_SCALEOFFSET
+ *-------------------------------------------------------------------------
+ */
+   else if (strcmp(scomp,"S+O")==0)
+   {
+    filt->filtn=H5Z_FILTER_SCALEOFFSET;
+    if (no_param) { /*no more parameters, S+O must have parameter */
+     if (obj_list) free(obj_list);
+     printf("Input Error: Missing compression parameter in <%s>\n",str);
+     exit(1);
+    }
+   }
    else {
     if (obj_list) free(obj_list);
     printf("Input Error: Invalid filter type in <%s>\n",str);
@@ -327,6 +341,13 @@ obj_list_t* parse_filter(const char *str,
    exit(1);
   }
   break;
+ case H5Z_FILTER_SCALEOFFSET:
+  if (filt->cd_values[0]<0 ){
+   if (obj_list) free(obj_list);
+   printf("Input Error: Invalid compression parameter in <%s>\n",str);
+   exit(1);
+  }
+  break;
  };
 
  return obj_list;
@@ -357,6 +378,8 @@ const char* get_sfilter(H5Z_filter_t filtn)
   return "FLETCHER32";
  else if (filtn==H5Z_FILTER_NBIT)
   return "NBIT";
+ else if (filtn==H5Z_FILTER_SCALEOFFSET)
+  return "S+O";
  else {
   printf("Input error in filter type\n");
   exit(1);
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 6c09dce..bc80c5a 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -1012,6 +1012,66 @@ if (szip_can_encode) {
 #else
  SKIPPED();
 #endif
+ TESTING("    copy of scaleoffset filter");
+ 
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();
+#else
+ SKIPPED();
+#endif
+
+  TESTING("    removing scaleoffset filter");
+
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();  
+#else
+ SKIPPED();
+#endif
+
+  TESTING("    adding scaleoffset filter");
+
+#ifdef H5_HAVE_FILTER_SCALEOFFSET
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset_none:S+O=31",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME13OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();  
+#else
+ SKIPPED();
+#endif
 
 /*-------------------------------------------------------------------------
  * file with all filters 
diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c
index 4245bec..9ae625d 100644
--- a/tools/h5repack/testh5repack_make.c
+++ b/tools/h5repack/testh5repack_make.c
@@ -34,6 +34,7 @@ int make_deflate(hid_t loc_id);
 int make_shuffle(hid_t loc_id);
 int make_fletcher32(hid_t loc_id);
 int make_nbit(hid_t loc_id);
+int make_scaleoffset(hid_t loc_id);
 int make_all(hid_t loc_id);
 int make_fill(hid_t loc_id);
 
@@ -168,6 +169,15 @@ int make_testfiles(void)
   goto out;
 
 /*-------------------------------------------------------------------------
+ * create a file with the scaleoffset filter
+ *-------------------------------------------------------------------------
+ */
+ if((loc_id = H5Fcreate(FNAME13,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
+  return -1;
+ if (make_scaleoffset(loc_id)<0)
+  goto out;
+
+/*-------------------------------------------------------------------------
  * create a file with the all filters
  *-------------------------------------------------------------------------
  */
@@ -735,6 +745,98 @@ out:
 
 
 /*-------------------------------------------------------------------------
+ * Function: make_scaleoffset
+ *
+ * Purpose: make a dataset with the scaleoffset filter
+ *
+ *-------------------------------------------------------------------------
+ */
+int make_scaleoffset(hid_t loc_id)
+{
+ hid_t    dcpl; /* dataset creation property list */
+ hid_t    sid;  /* dataspace ID */
+ hid_t    dtid;  
+ hid_t    dsid; 
+ hsize_t  dims[RANK]={DIM1,DIM2};
+ hsize_t  chunk_dims[RANK]={CDIM1,CDIM2};
+ int      buf[DIM1][DIM2];
+ int      i, j, n;
+
+ for (i=n=0; i<DIM1; i++){
+  for (j=0; j<DIM2; j++){
+   buf[i][j]=n++;
+  }
+ }
+ /* create a space */
+ if((sid = H5Screate_simple(RANK, dims, NULL))<0)
+  return -1;
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0)
+  goto out;
+ /* set up chunk */
+ if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
+  goto out;
+
+ dtid = H5Tcopy(H5T_NATIVE_INT);
+
+#if defined (H5_HAVE_FILTER_NBIT)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if (H5Pset_scaleoffset(dcpl,31)<0) 
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if((dsid = H5Dcreate (loc_id,"dset_scaleoffset",dtid,sid,dcpl))<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ H5Dclose(dsid);
+ if((dsid = H5Dcreate (loc_id,"dset_none",dtid,sid,H5P_DEFAULT))<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ H5Tclose(dtid);
+ H5Dclose(dsid);
+#endif
+
+/*-------------------------------------------------------------------------
+ * close space and dcpl
+ *-------------------------------------------------------------------------
+ */
+ if(H5Sclose(sid)<0)
+  goto out;
+ if(H5Pclose(dcpl)<0)
+  goto out;
+ 
+ return 0;                                                 
+ 
+out:
+ H5E_BEGIN_TRY {
+  H5Pclose(dcpl);
+  H5Sclose(sid);
+ } H5E_END_TRY;
+ return -1;
+}
+
+
+/*-------------------------------------------------------------------------
  * Function: make_all
  *
  * Purpose: make a file with all filters
diff --git a/tools/testfiles/test_scaleoffset.h5 b/tools/testfiles/test_scaleoffset.h5
new file mode 100644
index 0000000..4e78d83
--- /dev/null
+++ b/tools/testfiles/test_scaleoffset.h5