[svn-r10009] Purpose:

feature

Description:
support for nbit compression in h5repack

Solution:

Platforms tested:
verbena,copper,shanti

Misc. update:
manifest

8 files changed, 257 insertions, 1 deletions

diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index e4de4d3..a530b1b 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -275,6 +275,8 @@ int         parse_number(char *str);
 #define FNAME9OUT     "test_shuffle.out.h5"
 #define FNAME10OUT    "test_fletcher32.out.h5"
 #define FNAME11OUT    "test_all.out.h5"
+#define FNAME12    "test_nbit.h5"
+#define FNAME12OUT    "test_nbit.out.h5"
 
 int make_testfiles(void);
 
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index d1f00c1..1ca28b7 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -18,6 +18,7 @@ USE_FILTER_SZIP="@USE_FILTER_SZIP@"
 USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
 USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
 USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
+USE_FILTER_NBIT="@USE_FILTER_NBIT@"
 
 
 H5REPACK=h5repack               # The tool name
@@ -282,9 +283,33 @@ else
  TOOLTEST $arg
 fi        
 
+# nbit copy
+arg="test_nbit.h5"
+if test $USE_FILTER_NBIT != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
+# nbit remove
+arg="test_nbit.h5 -f dset_nbit:NONE"
+if test $USE_FILTER_NBIT != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi        
+
+# nbit add
+arg="test_nbit.h5 -f dset_int31:NBIT"
+if test $USE_FILTER_NBIT != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+
 # remove all  filters
 arg="test_all.h5 -f NONE"
-if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes"  ; then
+if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes"  -o $USE_FILTER_NBIT != "yes"  ; then
  SKIP $arg
 else
  TOOLTEST $arg
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 42b6701..3f558fc 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -115,6 +115,9 @@ static void print_obj(hid_t dcpl_id, char *name)
   case H5Z_FILTER_FLETCHER32:
    strcat(str,"FLET ");
    break;
+  case H5Z_FILTER_NBIT:
+   strcat(str,"NBIT ");
+   break;
   } /* switch */
  }/*i*/
 
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 89490e3..c9d6e20 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -287,6 +287,7 @@ int apply_filters(const char* name,    /* object name from traverse list */
  * H5Z_FILTER_SHUFFLE    2 , shuffle the data
  * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
  * H5Z_FILTER_SZIP       4 , szip compression 
+ * H5Z_FILTER_NBIT       5 , nbit compression 
  *-------------------------------------------------------------------------
  */
 	
@@ -369,6 +370,16 @@ int apply_filters(const char* name,    /* object name from traverse list */
 				if (H5Pset_fletcher32(dcpl_id)<0) 
 					return -1;
 				break;
+				/*-------------------------------------------------------------------------
+				* H5Z_FILTER_NBIT , NBIT compression
+				*-------------------------------------------------------------------------
+				*/
+			case H5Z_FILTER_NBIT:
+				if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
+					return -1;
+				if (H5Pset_nbit(dcpl_id)<0) 
+					return -1;
+				break;
 			} /* switch */
 		}/*i*/
 		
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index d9c5e3a..8771328 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -147,10 +147,12 @@ void usage(void)
  printf("       SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n");
  printf("       SHUF, to apply the HDF5 shuffle filter\n");
  printf("       FLET, to apply the HDF5 checksum filter\n");
+ printf("       NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n");
  printf("       NONE, to remove the filter\n");
  printf("     <filter parameters> is optional compression info\n");
  printf("       SHUF (no parameter)\n");
  printf("       FLET (no parameter)\n");
+ printf("       NBIT (no parameter)\n");
  printf("       GZIP=<deflation level> from 1-9\n");
  printf("       SZIP=<pixels per block,coding> (pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN')\n");
  printf("[-l 'layout']     Layout type. 'layout' is a string with the format\n");
diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c
index 8f548b0..9cd9b7d 100644
--- a/tools/h5repack/h5repack_parse.c
+++ b/tools/h5repack/h5repack_parse.c
@@ -34,6 +34,7 @@
  *  SZIP, to apply the HDF5 SZIP filter (SZIP compression)
  *  SHUF, to apply the HDF5 shuffle filter
  *  FLET, to apply the HDF5 checksum filter
+ *  NBIT, to apply the HDF5 NBIT filter (NBIT compression)
  *  NONE, to remove the filter 
  *
  * Examples: 
@@ -270,6 +271,19 @@ obj_list_t* parse_filter(const char *str,
      exit(1);
     }
    }
+/*-------------------------------------------------------------------------
+ * H5Z_FILTER_NBIT
+ *-------------------------------------------------------------------------
+ */
+   else if (strcmp(scomp,"NBIT")==0)
+   {
+    filt->filtn=H5Z_FILTER_NBIT;
+    if (m>0){ /*nbit does not have parameter */
+     if (obj_list) free(obj_list);
+     printf("Input Error: Extra parameter in NBIT <%s>\n",str);
+     exit(1);
+    }
+   }
    else {
     if (obj_list) free(obj_list);
     printf("Input Error: Invalid filter type in <%s>\n",str);
@@ -341,6 +355,8 @@ const char* get_sfilter(H5Z_filter_t filtn)
   return "SHUFFLE";
  else if (filtn==H5Z_FILTER_FLETCHER32)
   return "FLETCHER32";
+ else if (filtn==H5Z_FILTER_NBIT)
+  return "NBIT";
  else {
   printf("Input error in filter type\n");
   exit(1);
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 6729174..6c09dce 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -952,6 +952,67 @@ if (szip_can_encode) {
 #endif
 
  
+ TESTING("    copy of nbit filter");
+ 
+#ifdef H5_HAVE_FILTER_NBIT
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME12,FNAME12OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME12OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();
+#else
+ SKIPPED();
+#endif
+
+  TESTING("    removing nbit filter");
+
+#ifdef H5_HAVE_FILTER_NBIT
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset_nbit:NONE",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME12,FNAME12OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME12OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();  
+#else
+ SKIPPED();
+#endif
+
+  TESTING("    adding nbit filter");
+
+#ifdef H5_HAVE_FILTER_NBIT
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset_int31:NBIT",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME12,FNAME12OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME12OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+ PASSED();  
+#else
+ SKIPPED();
+#endif
+
 /*-------------------------------------------------------------------------
  * file with all filters 
  *  dset_all
diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c
index 65a1c4a..4245bec 100644
--- a/tools/h5repack/testh5repack_make.c
+++ b/tools/h5repack/testh5repack_make.c
@@ -33,6 +33,7 @@ int make_szip(hid_t loc_id);
 int make_deflate(hid_t loc_id);
 int make_shuffle(hid_t loc_id);
 int make_fletcher32(hid_t loc_id);
+int make_nbit(hid_t loc_id);
 int make_all(hid_t loc_id);
 int make_fill(hid_t loc_id);
 
@@ -158,6 +159,15 @@ int make_testfiles(void)
   return -1;
 
 /*-------------------------------------------------------------------------
+ * create a file with the nbit filter
+ *-------------------------------------------------------------------------
+ */
+ if((loc_id = H5Fcreate(FNAME12,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
+  return -1;
+ if (make_nbit(loc_id)<0)
+  goto out;
+
+/*-------------------------------------------------------------------------
  * create a file with the all filters
  *-------------------------------------------------------------------------
  */
@@ -622,6 +632,106 @@ out:
 }
 
 
+/*-------------------------------------------------------------------------
+ * Function: make_nbit
+ *
+ * Purpose: make a dataset with the nbit filter
+ *
+ *-------------------------------------------------------------------------
+ */
+int make_nbit(hid_t loc_id)
+{
+ hid_t    dcpl; /* dataset creation property list */
+ hid_t    sid;  /* dataspace ID */
+ hid_t    dtid;  
+ hid_t    dsid; 
+ hsize_t  dims[RANK]={DIM1,DIM2};
+ hsize_t  chunk_dims[RANK]={CDIM1,CDIM2};
+ int      buf[DIM1][DIM2];
+ int      i, j, n;
+
+ for (i=n=0; i<DIM1; i++){
+  for (j=0; j<DIM2; j++){
+   buf[i][j]=n++;
+  }
+ }
+ /* create a space */
+ if((sid = H5Screate_simple(RANK, dims, NULL))<0)
+  return -1;
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0)
+  goto out;
+ /* set up chunk */
+ if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
+  goto out;
+
+ dtid = H5Tcopy(H5T_NATIVE_INT);
+ if (H5Tset_precision(dtid,(H5Tget_precision(dtid) - 1)) < 0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+
+#if defined (H5_HAVE_FILTER_NBIT)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if (H5Pset_nbit(dcpl)<0) 
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if((dsid = H5Dcreate (loc_id,"dset_nbit",dtid,sid,dcpl))<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ H5Dclose(dsid);
+
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if((dsid = H5Dcreate (loc_id,"dset_int31",dtid,sid,dcpl))<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+ {
+  H5Tclose(dtid);
+  goto out;
+ }
+ H5Dclose(dsid);
+#endif
+
+/*-------------------------------------------------------------------------
+ * close space and dcpl
+ *-------------------------------------------------------------------------
+ */
+ if(H5Sclose(sid)<0)
+  goto out;
+ if(H5Pclose(dcpl)<0)
+  goto out;
+ 
+ return 0;                                                 
+ 
+out:
+ H5E_BEGIN_TRY {
+  H5Pclose(dcpl);
+  H5Sclose(sid);
+ } H5E_END_TRY;
+ return -1;
+}
 
 
 /*-------------------------------------------------------------------------
@@ -635,6 +745,8 @@ int make_all(hid_t loc_id)
 {
  hid_t    dcpl; /* dataset creation property list */
  hid_t    sid;  /* dataspace ID */
+ hid_t    dtid;  
+ hid_t    dsid;  
 #if defined (H5_HAVE_FILTER_SZIP)
  unsigned szip_options_mask=H5_SZIP_ALLOW_K13_OPTION_MASK|H5_SZIP_NN_OPTION_MASK;
  unsigned szip_pixels_per_block=8;
@@ -762,6 +874,30 @@ if (szip_can_encode) {
 #endif
 
 
+/*-------------------------------------------------------------------------
+ * nbit
+ *-------------------------------------------------------------------------
+ */
+#if defined (H5_HAVE_FILTER_NBIT)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
+  goto out;
+ /* set the shuffle filter */
+ if (H5Pset_nbit(dcpl)<0) 
+  goto out;
+ dtid = H5Tcopy(H5T_NATIVE_INT);
+ H5Tset_precision(dtid,(H5Tget_precision(dtid)-1));
+ if((dsid = H5Dcreate (loc_id,"dset_nbit",dtid,sid,dcpl))<0)
+  goto out;
+ if(H5Dwrite(dsid,dtid,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+  goto out;
+
+ /* close */
+ if(H5Tclose(dtid)<0)
+  return -1;
+ if(H5Dclose(dsid)<0)
+  return -1;
+#endif
 
 /*-------------------------------------------------------------------------
  * close space and dcpl