Merge pull request #543 in HDFFV/hdf5 from ~BYRN/hdf5_adb:hdf5_1_8 to hdf5_1_8

* commit 'c13e0835f70b3ba9d9d706d772b5fa0f2a534cbf':
  Add missing creation order flag set
  Update usage reference file for test
  Remove unavailable argument
  Add missing variable
  HDFFV-8611 preserve creation order for root group

14 files changed, 169 insertions, 48 deletions

diff --git a/tools/h5dump/h5dump.h b/tools/h5dump/h5dump.h
index 757087d..cf4c8c1 100644
--- a/tools/h5dump/h5dump.h
+++ b/tools/h5dump/h5dump.h
@@ -76,10 +76,6 @@ int          disable_compact_subset= FALSE; /* disable compact form of subset no
 int          display_packed_bits = FALSE; /*print 1-8 byte numbers as packed bits*/
 int          include_attrs     = TRUE; /* Display attributes */
 
-/* sort parameters */
-H5_index_t   sort_by           = H5_INDEX_NAME; /*sort_by [creation_order | name]  */
-H5_iter_order_t sort_order     = H5_ITER_INC; /*sort_order [ascending | descending]   */
-
 #define PACKED_BITS_MAX         8  /* Maximum number of packed-bits to display */
 #define PACKED_BITS_SIZE_MAX    (8*sizeof(long long))  /* Maximum bits size of integer types of packed-bits */
 /* mask list for packed bits */
diff --git a/tools/h5dump/h5dump_extern.h b/tools/h5dump/h5dump_extern.h
index 49816ac..ae65174 100644
--- a/tools/h5dump/h5dump_extern.h
+++ b/tools/h5dump/h5dump_extern.h
@@ -75,10 +75,6 @@ extern int          disable_compact_subset; /* disable compact form of subset no
 extern int          display_packed_bits; /*print 1-8 byte numbers as packed bits*/
 extern int          include_attrs; /* Display attributes */
 
-/* sort parameters */
-extern H5_index_t   sort_by; /*sort_by [creation_order | name]  */
-extern H5_iter_order_t sort_order; /*sort_order [ascending | descending]   */
-
 #define PACKED_BITS_MAX         8  /* Maximum number of packed-bits to display */
 #define PACKED_BITS_SIZE_MAX    8*sizeof(long long)  /* Maximum bits size of integer types of packed-bits */
 /* mask list for packed bits */
diff --git a/tools/h5repack/CMakeTests.cmake b/tools/h5repack/CMakeTests.cmake
index 3710202..dafed86 100644
--- a/tools/h5repack/CMakeTests.cmake
+++ b/tools/h5repack/CMakeTests.cmake
@@ -88,6 +88,7 @@
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00008.h5
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00009.h5
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00010.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/tordergr.h5
   )
 
   set (LIST_OTHER_TEST_FILES
@@ -106,6 +107,7 @@
       ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_version_test.ddl
       ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/plugin_version_test.h5repack_layout.h5.tst
       ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst
+      ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl
   )
 
   foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES})
@@ -247,7 +249,7 @@
             NAME H5REPACK_DMP-h5dump-${testname}
             COMMAND "${CMAKE_COMMAND}"
                 -D "TEST_PROGRAM=$<TARGET_FILE:h5dump>"
-                -D "TEST_ARGS:STRING=-pH;out-${testname}.${resultfile}"
+                -D "TEST_ARGS:STRING=-q;creation_order;-pH;out-${testname}.${resultfile}"
                 -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
                 -D "TEST_OUTPUT=${resultfile}-${testname}.out"
                 -D "TEST_EXPECT=${resultcode}"
@@ -836,6 +838,11 @@
   endif ()
   ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg})
 
+#crtorder
+  set (arg tordergr.h5 -L)
+  set (TESTTYPE "TEST")
+  ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg})
+
 #########################################################
 # layout options (these files have no filters)
 #########################################################
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 35cf7a0..0ab5404 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -780,6 +780,8 @@ static int check_objects(const char* fname, pack_opt_t *options) {
     *-------------------------------------------------------------------------
     */
 
+    /* Initialize indexing options */
+    h5trav_set_index(sort_by, sort_order);
     /* init table */
     trav_table_init(&travt);
 
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 3b18185..3f5e013 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -113,6 +113,7 @@ $SRC_TOOLS_TESTFILES/tfamily00007.h5
 $SRC_TOOLS_TESTFILES/tfamily00008.h5
 $SRC_TOOLS_TESTFILES/tfamily00009.h5
 $SRC_TOOLS_TESTFILES/tfamily00010.h5
+$SRC_TOOLS_TESTFILES/tordergr.h5
 "
 
 LIST_OTHER_TEST_FILES="
@@ -120,6 +121,7 @@ $SRC_H5REPACK_TESTFILES/h5repack-help.txt
 $SRC_H5REPACK_TESTFILES/h5repack_ext.bin
 $SRC_H5REPACK_TESTFILES/ublock.bin
 $SRC_H5REPACK_TESTFILES/h5repack.info
+$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
 $SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
@@ -531,7 +533,7 @@ TOOLTEST_DUMP()
      VERIFY h5dump output $@
      (
         cd $TESTDIR
-        $RUNSERIAL $H5DUMP_BIN -pH $outfile
+        $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile
      ) >$actual 2>$actual_err
      cat $actual_err >> $actual
 
@@ -898,6 +900,10 @@ else
  TOOLTEST deflate_file $arg
 fi
 
+#crtorder
+arg="tordergr.h5 -L"
+TOOLTEST_DUMP crtorder $arg
+
 #########################################################
 # layout options (these files have no filters)
 #########################################################
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 40c3888..b41f7be 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -95,6 +95,7 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
     hsize_t       ub_size = 0;            /* size of user block */
     hid_t         fcpl = H5P_DEFAULT;     /* file creation property list ID */
     hid_t         fapl = H5P_DEFAULT;     /* file access property list ID */
+    unsigned      crt_order_flags;        /* group creation order flag */
 
     /*-------------------------------------------------------------------------
      * open input file
@@ -108,6 +109,8 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
     /* get user block size and file space strategy/threshold */
     {
         hid_t fcpl_in; /* file creation property list ID for input file */
+        hid_t grp_in = -1;   /* group ID */
+        hid_t gcpl_in = -1;  /* group creation property list */
 
         if ((fcpl_in = H5Fget_create_plist(fidin)) < 0) {
             error_msg("failed to retrieve file creation property list\n");
@@ -119,6 +122,18 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
         }
 
+        /* open root group */
+        if ((grp_in = H5Gopen2(fidin, "/", H5P_DEFAULT)) < 0)
+            HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed");
+
+        /* get root group creation property list */
+        if ((gcpl_in = H5Gget_create_plist(grp_in)) < 0)
+            HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed");
+
+        /* query and set the group creation properties */
+        if (H5Pget_link_creation_order(gcpl_in, &crt_order_flags) < 0)
+            HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed");
+
         if (H5Pclose(fcpl_in) < 0) {
             error_msg("failed to close property list\n");
             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
@@ -289,6 +304,9 @@ print_user_block(fnamein, fidin);
         }
     }
 
+    if(H5Pset_link_creation_order(fcpl, crt_order_flags ) < 0)
+        HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_creation_order failed");
+
     /*-------------------------------------------------------------------------
      * create the output file
      *-------------------------------------------------------------------------
@@ -317,6 +335,8 @@ print_user_block(fnamein, fidin);
      *-------------------------------------------------------------------------
      */
 
+    /* Initialize indexing options */
+    h5trav_set_index(sort_by, sort_order);
     /* init table */
     trav_table_init(&travt);
 
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index d14b604..e1f13fd 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -31,7 +31,7 @@ const char *outfile = NULL;
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:E";
+static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:q:z:E";
 static struct long_options l_opts[] = {
     { "help", no_arg, 'h' },
     { "version", no_arg, 'V' },
@@ -52,6 +52,8 @@ static struct long_options l_opts[] = {
     { "alignment", require_arg, 'a' },
     { "infile", require_arg, 'i' }, /* -i for backward compability */
     { "outfile", require_arg, 'o' }, /* -o for backward compability */
+    { "sort_by", require_arg, 'q' },
+    { "sort_order", require_arg, 'z' },
     { "enable-error-stack", no_arg, 'E' },
     { NULL, 0, '\0' }
 };
@@ -86,6 +88,8 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "   -M A, --metadata_block_size=A  Metadata block size for H5Pset_meta_block_size\n");
     PRINTVALSTREAM(rawoutstream, "   -t T, --threshold=T     Threshold value for H5Pset_alignment\n");
     PRINTVALSTREAM(rawoutstream, "   -a A, --alignment=A     Alignment value for H5Pset_alignment\n");
+    PRINTVALSTREAM(rawoutstream, "   -q Q, --sort_by=Q       Sort groups and attributes by index Q\n");
+    PRINTVALSTREAM(rawoutstream, "   -z Z, --sort_order=Z    Sort groups and attributes by order Z\n");
     PRINTVALSTREAM(rawoutstream, "   -f FILT, --filter=FILT  Filter type\n");
     PRINTVALSTREAM(rawoutstream, "   -l LAYT, --layout=LAYT  Layout type\n");
     PRINTVALSTREAM(rawoutstream, "\n");
@@ -95,6 +99,8 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "    U - is a filename.\n");
     PRINTVALSTREAM(rawoutstream, "    T - is an integer\n");
     PRINTVALSTREAM(rawoutstream, "    A - is an integer greater than zero\n");
+    PRINTVALSTREAM(rawoutstream, "    Q - is the sort index type for the input file. It can be \"name\" or \"creation_order\" (default)\n");
+    PRINTVALSTREAM(rawoutstream, "    Z - is the sort order type for the input file. It can be \"descending\" or \"ascending\" (default)\n");
     PRINTVALSTREAM(rawoutstream, "    B - is the user block size, any value that is 512 or greater and is\n");
     PRINTVALSTREAM(rawoutstream, "        a power of 2 (1024 default)\n");
     PRINTVALSTREAM(rawoutstream, "    F - is the shared object header message type, any of <dspace|dtype|fill|\n");
@@ -340,6 +346,52 @@ done:
 }
 
 /*-------------------------------------------------------------------------
+ * Function:    set_sort_by
+ *
+ * Purpose: set the "by" form of sorting by translating from a string input
+ *          parameter to a H5_index_t return value
+ *          current sort values are [creation_order | name]
+ *
+ * Return: H5_index_t form of sort or H5_INDEX_UNKNOWN if none found
+ *-------------------------------------------------------------------------
+ */
+static H5_index_t
+set_sort_by(const char *form)
+{
+    H5_index_t idx_type = H5_INDEX_UNKNOWN;
+
+    if (HDstrcmp(form,"name")==0) /* H5_INDEX_NAME */
+        idx_type = H5_INDEX_NAME;
+    else if (HDstrcmp(form,"creation_order")==0) /* H5_INDEX_CRT_ORDER */
+        idx_type = H5_INDEX_CRT_ORDER;
+
+    return idx_type;
+}
+
+/*-------------------------------------------------------------------------
+ * Function:    set_sort_order
+ *
+ * Purpose: set the order of sorting by translating from a string input
+ *          parameter to a H5_iter_order_t return value
+ *          current order values are [ascending | descending ]
+ *
+ * Return: H5_iter_order_t form of order or H5_ITER_UNKNOWN if none found
+ *-------------------------------------------------------------------------
+ */
+static H5_iter_order_t
+set_sort_order(const char *form)
+{
+    H5_iter_order_t iter_order = H5_ITER_UNKNOWN;
+
+    if (HDstrcmp(form,"ascending")==0) /* H5_ITER_INC */
+        iter_order = H5_ITER_INC;
+    else if (HDstrcmp(form,"descending")==0) /* H5_ITER_DEC */
+        iter_order = H5_ITER_DEC;
+
+    return iter_order;
+}
+
+/*-------------------------------------------------------------------------
  * Function: parse_command_line
  *
  * Purpose: parse command line input
@@ -497,9 +549,27 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
                 }
                 break;
 
-	    case 'E':
-		enable_error_stack = TRUE;
-		break;
+            case 'q':
+                if((sort_by = set_sort_by(opt_arg)) < 0) {
+                    error_msg(" failed to set sort by form <%s>\n", opt_arg);
+                    h5tools_setstatus(EXIT_FAILURE);
+                    ret_value = -1;
+                    goto done;
+                }
+                break;
+
+            case 'z':
+                if((sort_order = set_sort_order(opt_arg)) < 0) {
+                    error_msg(" failed to set sort order form <%s>\n", opt_arg);
+                    h5tools_setstatus(EXIT_FAILURE);
+                    ret_value = -1;
+                    goto done;
+                }
+                break;
+
+            case 'E':
+            enable_error_stack = TRUE;
+            break;
 
             default:
                 break;
@@ -568,6 +638,9 @@ int main(int argc, const char **argv)
     /* initialize options  */
     h5repack_init(&options, 0);
 
+    /* Initialize default indexing options */
+    sort_by = H5_INDEX_CRT_ORDER;
+
     if (parse_command_line(argc, argv, &options) < 0)
         goto done;
 
diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c
index 3905842..003875d 100644
--- a/tools/h5repack/h5repack_verify.c
+++ b/tools/h5repack/h5repack_verify.c
@@ -115,6 +115,8 @@ h5repack_verify(const char *out_fname, pack_opt_t *options)
     if(options->all_filter == 1 || options->all_layout == 1)
     {
 
+        /* Initialize indexing options */
+        h5trav_set_index(sort_by, sort_order);
         /* init table */
         trav_table_init(&travt);
 
@@ -329,6 +331,9 @@ int h5repack_cmp_pl(const char *fname1,
     * get file table list of objects
     *-------------------------------------------------------------------------
     */
+    /* Initialize indexing options */
+    h5trav_set_index(sort_by, sort_order);
+    /* init table */
     trav_table_init(&trav);
     if(h5trav_gettable(fid1, trav) < 0)
         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed");
@@ -517,7 +522,7 @@ int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter)
         {
 
             case H5Z_FILTER_NONE:
-        	break;
+            break;
 
             case H5Z_FILTER_SHUFFLE:
                 /* 1 private client value is returned by DCPL */
diff --git a/tools/h5repack/testfiles/crtorder.tordergr.h5.ddl b/tools/h5repack/testfiles/crtorder.tordergr.h5.ddl
new file mode 100644
index 0000000..870d154
--- /dev/null
+++ b/tools/h5repack/testfiles/crtorder.tordergr.h5.ddl
@@ -0,0 +1,36 @@
+HDF5 "out-crtorder.tordergr.h5" {
+GROUP "/" {
+   GROUP "2" {
+      GROUP "a" {
+         GROUP "a1" {
+         }
+         GROUP "a2" {
+            GROUP "a21" {
+            }
+            GROUP "a22" {
+            }
+         }
+      }
+      GROUP "b" {
+      }
+      GROUP "c" {
+      }
+   }
+   GROUP "1" {
+      GROUP "c" {
+      }
+      GROUP "b" {
+      }
+      GROUP "a" {
+         GROUP "a1" {
+         }
+         GROUP "a2" {
+            GROUP "a22" {
+            }
+            GROUP "a21" {
+            }
+         }
+      }
+   }
+}
+}
diff --git a/tools/h5repack/testfiles/h5repack-help.txt b/tools/h5repack/testfiles/h5repack-help.txt
index 5c30ce8..b9157d7 100644
--- a/tools/h5repack/testfiles/h5repack-help.txt
+++ b/tools/h5repack/testfiles/h5repack-help.txt
@@ -17,6 +17,8 @@ usage: h5repack [OPTIONS] file1 file2
    -M A, --metadata_block_size=A  Metadata block size for H5Pset_meta_block_size
    -t T, --threshold=T     Threshold value for H5Pset_alignment
    -a A, --alignment=A     Alignment value for H5Pset_alignment
+   -q Q, --sort_by=Q       Sort groups and attributes by index Q
+   -z Z, --sort_order=Z    Sort groups and attributes by order Z
    -f FILT, --filter=FILT  Filter type
    -l LAYT, --layout=LAYT  Layout type
 
@@ -26,6 +28,8 @@ usage: h5repack [OPTIONS] file1 file2
     U - is a filename.
     T - is an integer
     A - is an integer greater than zero
+    Q - is the sort index type for the input file. It can be "name" or "creation_order" (default)
+    Z - is the sort order type for the input file. It can be "descending" or "ascending" (default)
     B - is the user block size, any value that is 512 or greater and is
         a power of 2 (1024 default)
     F - is the shared object header message type, any of <dspace|dtype|fill|
diff --git a/tools/lib/h5diff_attr.c b/tools/lib/h5diff_attr.c
index 004672e..3018f18 100644
--- a/tools/lib/h5diff_attr.c
+++ b/tools/lib/h5diff_attr.c
@@ -91,7 +91,7 @@ static void table_attrs_free( table_attrs_t *table )
 /*-------------------------------------------------------------------------
  * Function: table_attr_mark_exist
  *
- * Purpose: mark given attribute name to table as sign of exsit
+ * Purpose: mark given attribute name to table as sign of exist
  *
  * Parameter:
  *  - exist [IN]
diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
index 9a7dc9e..159349a 100644
--- a/tools/lib/h5tools.c
+++ b/tools/lib/h5tools.c
@@ -51,6 +51,10 @@ unsigned    packed_data_length; /* length of packed bits to display */
 unsigned long long packed_data_mask;  /* mask in which packed bits to display */
 int          enable_error_stack= FALSE; /* re-enable error stack */
 
+/* sort parameters */
+H5_index_t   sort_by           = H5_INDEX_NAME; /*sort_by [creation_order | name]  */
+H5_iter_order_t sort_order     = H5_ITER_INC; /*sort_order [ascending | descending]   */
+
 /* module-scoped variables */
 static int  h5tools_init_g;     /* if h5tools lib has been initialized */
 
diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h
index 2e01ad2..09ec527 100644
--- a/tools/lib/h5tools.h
+++ b/tools/lib/h5tools.h
@@ -540,6 +540,10 @@ H5TOOLS_DLLVAR int     oid_output;          /* oid output */
 H5TOOLS_DLLVAR int     data_output;         /* data output */
 H5TOOLS_DLLVAR int     attr_data_output;    /* attribute data output */
 
+/* sort parameters */
+H5TOOLS_DLLVAR H5_index_t   sort_by;        /*sort_by [creation_order | name]  */
+H5TOOLS_DLLVAR H5_iter_order_t sort_order;  /*sort_order [ascending | descending]   */
+
 /* things to display or which are set via command line parameters */
 H5TOOLS_DLLVAR int     enable_error_stack; /* re-enable error stack */
 
diff --git a/tools/lib/h5tools_type.c b/tools/lib/h5tools_type.c
index b57e274..ae7160b 100644
--- a/tools/lib/h5tools_type.c
+++ b/tools/lib/h5tools_type.c
@@ -15,38 +15,6 @@
 #include "h5tools.h"
 
 /*-------------------------------------------------------------------------
- * Function: h5tools_get_native_type
- *
- * Purpose: Wrapper around H5Tget_native_type() to work around
- *  Problems with bitfields.
- *
- * Return: Success:    datatype ID
- *
- *  Failure:    FAIL
- *
- * Programmer: Quincey Koziol
- *              Tuesday, October  5, 2004
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-hid_t
-h5tools_get_native_type(hid_t type)
-{
-    hid_t p_type;
-    H5T_class_t type_class;
-
-    type_class = H5Tget_class(type);
-    if(type_class==H5T_BITFIELD)
-        p_type=H5Tcopy(type);
-    else
-        p_type = H5Tget_native_type(type,H5T_DIR_DEFAULT);
-
-    return(p_type);
-}
-
-/*-------------------------------------------------------------------------
  * Function: h5tools_get_little_endian_type
  *
  * Purpose: Get a little endian type from a file type