HDFFV-10276 Add UD flag option to h5repack

9 files changed, 45 insertions, 36 deletions

diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt
index e2489f4..9c0c91e 100644
--- a/release_docs/INSTALL_CMake.txt
+++ b/release_docs/INSTALL_CMake.txt
@@ -656,17 +656,6 @@ The same scripts can be used on Linux, Mac OSX or a Windows machine by
 adding an option (${CTEST_SCRIPT_ARG}) to the platform configuration script.
 
 
-#
-# Copyright by The HDF Group.
-# All rights reserved.
-#
-# This file is part of HDF5.  The full HDF5 copyright notice, including
-# terms governing use, modification, and redistribution, is contained in
-# the COPYING file, which can be found at the root of the source code
-# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
-# If you do not have access to either file, you may request a copy from
-# help@hdfgroup.org.
-#
 #############################################################################################
 ### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE                                       ###
 ### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ###
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 64015ba..d0de3ee 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -107,7 +107,6 @@ New Features
 
       (ADB - 2017/08/30, HDFFV-9774)
 
-
     High-Level APIs:
     ---------------
     -
@@ -157,6 +156,15 @@ Bug Fixes since HDF5-1.10.1 release
 
     Tools
     -----
+    - h5repack
+
+      h5repack always set the User Defined filter flag to H5Z_FLAG_MANDATORY.
+
+      Added another parameter to the 'UD=' option to set the flag by default
+      to '0' or H5Z_FLAG_MANDATORY, the other choice is '1' or H5Z_FLAG_OPTIONAL.
+
+      (ADB - 2017/08/31, HDFFV-10276)
+
     - h5ls
 
       h5ls generated error on stack when it encountered a H5S_NULL
diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h
index e5ae03f..c8c6a0e 100644
--- a/tools/src/h5repack/h5repack.h
+++ b/tools/src/h5repack/h5repack.h
@@ -61,6 +61,7 @@ typedef struct {
 
 typedef struct {
  H5Z_filter_t filtn;                           /* filter identification number */
+ unsigned     filt_flag;                       /* filter definition flag */
  unsigned     cd_values[CD_VALUES];            /* filter client data values */
  size_t       cd_nelmts;                       /* filter client number of values */
 } filter_info_t;
diff --git a/tools/src/h5repack/h5repack_filters.c b/tools/src/h5repack/h5repack_filters.c
index c48158c..5ccb2ef 100644
--- a/tools/src/h5repack/h5repack_filters.c
+++ b/tools/src/h5repack/h5repack_filters.c
@@ -402,7 +402,7 @@ int apply_filters(const char* name, /* object name from traverse list */
             default:
                 {
                     if (H5Pset_filter(dcpl_id, obj.filter[i].filtn,
-                            H5Z_FLAG_MANDATORY, obj.filter[i].cd_nelmts,
+                            obj.filter[i].filt_flag, obj.filter[i].cd_nelmts,
                             obj.filter[i].cd_values) < 0)
                         return -1;
                     if (H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths) < 0)
diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c
index 31b1e14..136d550 100644
--- a/tools/src/h5repack/h5repack_main.c
+++ b/tools/src/h5repack/h5repack_main.c
@@ -101,7 +101,7 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "   -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD   Free-space section threshold for H5Pset_file_space_strategy\n");
     PRINTVALSTREAM(rawoutstream, "   -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE   File space page size for H5Pset_file_space_page_size\n");
     PRINTVALSTREAM(rawoutstream, "\n");
-    PRINTVALSTREAM(rawoutstream, "    M - is an integer greater than 1, size of dataset in bytes (default is 0) \n");
+    PRINTVALSTREAM(rawoutstream, "    M - is an integer greater than 1, size of dataset in bytes (default is 0)\n");
     PRINTVALSTREAM(rawoutstream, "    E - is a filename.\n");
     PRINTVALSTREAM(rawoutstream, "    S - is an integer\n");
     PRINTVALSTREAM(rawoutstream, "    U - is a filename.\n");
@@ -164,8 +164,8 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "        NBIT (no parameter)\n");
     PRINTVALSTREAM(rawoutstream, "        SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n");
     PRINTVALSTREAM(rawoutstream, "            is either IN or DS\n");
-    PRINTVALSTREAM(rawoutstream, "        UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]>\n");
-    PRINTVALSTREAM(rawoutstream, "            required values for filter_number,cd_value_count,value_1\n");
+    PRINTVALSTREAM(rawoutstream, "        UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>\n");
+    PRINTVALSTREAM(rawoutstream, "            required values for filter_number,filter_flag,cd_value_count,value_1\n");
     PRINTVALSTREAM(rawoutstream, "            optional values for value_2 to value_N\n");
     PRINTVALSTREAM(rawoutstream, "        NONE (no parameter)\n");
     PRINTVALSTREAM(rawoutstream, "\n");
@@ -209,7 +209,7 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "\n");
     PRINTVALSTREAM(rawoutstream, "   Add both filters SHUF and GZIP in this order to all datasets\n");
     PRINTVALSTREAM(rawoutstream, "\n");
-    PRINTVALSTREAM(rawoutstream, "6) h5repack -f UD=307,1,9 file1 file2 \n");
+    PRINTVALSTREAM(rawoutstream, "6) h5repack -f UD=307,0,1,9 file1 file2 \n");
     PRINTVALSTREAM(rawoutstream, "\n");
     PRINTVALSTREAM(rawoutstream, "   Add bzip2 filter to all datasets\n");
     PRINTVALSTREAM(rawoutstream, "\n");
diff --git a/tools/src/h5repack/h5repack_parse.c b/tools/src/h5repack/h5repack_parse.c
index a9a890d..4d62723 100644
--- a/tools/src/h5repack/h5repack_parse.c
+++ b/tools/src/h5repack/h5repack_parse.c
@@ -42,7 +42,7 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
     size_t      i, m, u;
     char        c;
     size_t      len = HDstrlen(str);
-    int         k, l, p, q, end_obj = -1, no_param = 0;
+    int         f, k, l, p, q, end_obj = -1, no_param = 0;
     unsigned    j, n;
     char        sobj[MAX_NC_NAME];
     char        scomp[10] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
@@ -209,13 +209,14 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
 
                 /*-------------------------------------------------------------------------
                 * User Defined
-                *   has the format UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]>
+                *   has the format UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
                 *  BZIP2 example
-                *  UD=307,1,9
+                *  UD=307,0,1,9
                 *-------------------------------------------------------------------------
                 */
                 else if (HDstrcmp(scomp, "UD") == 0) {
                     l = -1; /* filter number index check */
+                    f = -1; /* filter flag index check */
                     p = -1; /* CD_VAL count check */
                     for (m = 0, q = 0, u = i + 1; u < len; u++, m++, q++) {
                         if (str[u] == ',') {
@@ -224,6 +225,10 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
                                 filt->filtn = HDatoi(stype);
                                 l = 0;
                             }
+                            else if (f == -1) {
+                                filt->filt_flag = HDstrtoul(stype, NULL, 0);
+                                f = 0;
+                            }
                             else if (p == -1) {
                                 filt->cd_nelmts = HDstrtoull(stype, NULL, 0);
                                 p = 0;
@@ -240,6 +245,12 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
                             error_msg("filter number parameter is not a digit in <%s>\n", str);
                             HDexit(EXIT_FAILURE);
                         }
+                        else if (!HDisdigit(c) && f == -1) {
+                            if (obj_list)
+                                HDfree(obj_list);
+                            error_msg("filter flag parameter is not a digit in <%s>\n", str);
+                            HDexit(EXIT_FAILURE);
+                        }
                         stype[q] = c;
                     } /* for u */
                     stype[q] = '\0';
diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake
index 225f6a8..dfdbb14 100644
--- a/tools/test/h5repack/CMakeTests.cmake
+++ b/tools/test/h5repack/CMakeTests.cmake
@@ -1256,15 +1256,15 @@
 ##############################################################################
 ###    P L U G I N  T E S T S
 ##############################################################################
-  ADD_H5_UD_TEST (plugin_version_test 0 h5repack_layout.h5 -v -f UD=260,4,9,${H5_VERS_MAJOR},${H5_VERS_MINOR},${H5_VERS_RELEASE})
-  ADD_H5_UD_TEST (plugin_test 0 h5repack_layout.h5 -v -f UD=257,1,9)
+  ADD_H5_UD_TEST (plugin_version_test 0 h5repack_layout.h5 -v -f UD=260,0,4,9,${H5_VERS_MAJOR},${H5_VERS_MINOR},${H5_VERS_RELEASE})
+  ADD_H5_UD_TEST (plugin_test 0 h5repack_layout.h5 -v -f UD=257,0,1,9)
   ADD_H5_UD_TEST (plugin_none 0 h5repack_layout.UD.h5 -v -f NONE)
   # check for no parameters
   set (TESTRETVAL 255)
   if (WIN32)
     set (TESTRETVAL -1)
   endif ()
-  ADD_H5_CMP_TEST (plugin_zero "" "TEST" ${TESTRETVAL} h5repack_layout.h5 -v -f UD=250,0)
+  ADD_H5_CMP_TEST (plugin_zero "" "TEST" ${TESTRETVAL} h5repack_layout.h5 -v -f UD=250,0,0)
 
   if (HDF5_TEST_VFD)
     # Run test with different Virtual File Driver
diff --git a/tools/test/h5repack/h5repack_plugin.sh.in b/tools/test/h5repack/h5repack_plugin.sh.in
index c9fe53c..5efd9e8 100644
--- a/tools/test/h5repack/h5repack_plugin.sh.in
+++ b/tools/test/h5repack/h5repack_plugin.sh.in
@@ -1,10 +1,10 @@
 #! /bin/sh
 #
-# Copyright by The HDF Group.                                              
-# All rights reserved.                                                     
-#                                                                          
-# This file is part of HDF5. The full HDF5 copyright notice, including     
-# terms governing use, modification, and redistribution, is contained in   
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
 # the COPYING file, which can be found at the root of the source code
 # distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
 # If you do not have access to either file, you may request a copy from
@@ -247,10 +247,10 @@ COPY_TESTFILES_TO_TESTDIR
 version_str=`echo @H5_VERSION@ | awk -F"-" '{print $1}' | sed 's/\./,/g'`
 
 # Run the test
-arg="h5repack_layout.h5 -v -f UD=260,4,9,$version_str"
+arg="h5repack_layout.h5 -v -f UD=260,0,4,9,$version_str"
 TOOLTEST_DUMP plugin_version_test $arg
 
-arg="h5repack_layout.h5 -v -f UD=257,1,9"
+arg="h5repack_layout.h5 -v -f UD=257,0,1,9"
 TOOLTEST_DUMP plugin_test $arg
 
 #arg="h5repack_layout.UD.h5 -v -f NONE"
diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt
index 65edd89..1ff5b73 100644
--- a/tools/test/h5repack/testfiles/h5repack-help.txt
+++ b/tools/test/h5repack/testfiles/h5repack-help.txt
@@ -26,7 +26,7 @@ usage: h5repack [OPTIONS] file1 file2
    -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD   Free-space section threshold for H5Pset_file_space_strategy
    -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE   File space page size for H5Pset_file_space_page_size
 
-    M - is an integer greater than 1, size of dataset in bytes (default is 0) 
+    M - is an integer greater than 1, size of dataset in bytes (default is 0)
     E - is a filename.
     S - is an integer
     U - is a filename.
@@ -89,8 +89,8 @@ usage: h5repack [OPTIONS] file1 file2
         NBIT (no parameter)
         SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type
             is either IN or DS
-        UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]>
-            required values for filter_number,cd_value_count,value_1
+        UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
+            required values for filter_number,filter_flag,cd_value_count,value_1
             optional values for value_2 to value_N
         NONE (no parameter)
 
@@ -125,16 +125,16 @@ Examples of use:
    Chunked layout, with a layout size of 20x10, to objects dset1 and dset2
    and remove filters to objects dset3, dset4, dset5
 
-4) h5repack -L -c 10 -s 20:dtype file1 file2 
+4) h5repack -L -c 10 -s 20:dtype file1 file2
 
    Using latest file format with maximum compact group size of 10 and
    and minimum shared datatype size of 20
 
-5) h5repack -f SHUF -f GZIP=1 file1 file2 
+5) h5repack -f SHUF -f GZIP=1 file1 file2
 
    Add both filters SHUF and GZIP in this order to all datasets
 
-6) h5repack -f UD=307,1,9 file1 file2 
+6) h5repack -f UD=307,0,1,9 file1 file2
 
    Add bzip2 filter to all datasets