[svn-r22216] Description:

    Bring r22180, r22181 and r22214 from trunk to 1.8 branch:

Add new "metadata block size" command line option ('-M <x>' or
    '--metadata_block_size=<x>') for h5repack.

    [With release managers blessing]

Tested on:
    Mac OSX/64 10.7.3 (amazon) w/debug)
    (h5committest upcoming)

6 files changed, 124 insertions, 86 deletions

diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 201c711..27662e9 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -85,12 +85,12 @@ int h5repack(const char* infile,
 *
 *-------------------------------------------------------------------------
 */
-
-int h5repack_init (pack_opt_t *options,
+int h5repack_init(pack_opt_t *options,
                    int verbose)
 {
     int k, n;
-    HDmemset(options,0,sizeof(pack_opt_t));
+
+    HDmemset(options, 0, sizeof(pack_opt_t));
     options->min_comp = 1024;
     options->verbose  = verbose;
 
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index 0e724ec..4c43bd4 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -111,6 +111,7 @@ typedef struct {
                                  datatype, fill value, filter pipleline, attribute */
  const char      *ublock_filename; /* user block file name */
  hsize_t         ublock_size;      /* user block size */
+ hsize_t         meta_block_size;  /* metadata aggregation block size (for H5Pset_meta_block_size) */
  hsize_t         threshold;        /* alignment threshold for H5Pset_alignment */
  hsize_t         alignment ;       /* alignment for H5Pset_alignment */
 } pack_opt_t;
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index c1cbba1..236c555 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -147,20 +147,25 @@ COPY_TESTFILES_TO_TESTDIR()
     done
 }
 
+# Print a $* message left justified in a field of 70 characters
+#
+MESSAGE() {
+   SPACES="                                                               "
+   echo "$* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
 # Print a line-line message left justified in a field of 70 characters
 # beginning with the word "Testing".
 #
 TESTING() {
-   SPACES="                                                               "
-   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+    MESSAGE "Testing $*"
 }
 
 # Print a line-line message left justified in a field of 70 characters
 # beginning with the word "Verifying".
 #
 VERIFY() {
- SPACES="                                                               "
- echo "Verifying h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012'
+    MESSAGE "Verifying h5diff output $*"
 }
 
 # Print a message that a test has been skipped (because a required filter
@@ -385,6 +390,49 @@ TOOLTESTV()
    rm -f $actual $actual_err $actual_sav
 }
 
+# TOOLTEST_META:
+# Test metadata block size option.
+# Reason to create a function here is to localize all special steps related to
+# metadata block size option in one place.  This is a quick solution.  More
+# thought out solution needed when there is more time.
+#
+# $1: test input file
+# $2:$: metadata options (either "-M size" or "--metadata_block_size=size")
+#
+# Algorithm:
+# Run it once without the metadata option ($2-$);
+# Save the result output file;
+# Run it second time with the metadata option;
+# Verify the output file of second run is larger than the one of 1st run.
+TOOLTEST_META()
+{
+    input_file=$1
+    outfile="$TESTDIR/out.$1"
+
+    # Use TOOLTEST_MAIN to run because it does not remove the output file.
+    # 1st run, without metadata option
+    TOOLTEST_MAIN $1
+    # get the size of the first output file
+    size1=`wc -c $outfile | cut -d' ' -f1`
+
+    # 2nd run with metadata option
+    TOOLTEST_MAIN $*
+    # get the size of the second output file
+    size2=`wc -c $outfile | cut -d' ' -f1`
+
+    # verify sizes.
+    MESSAGE "Verify the sizes of both output files ($size1 vs $size2)"
+    if [ $size1 -lt $size2 ]; then
+    	# pass
+	echo " PASSED"
+    else
+	#fail
+	echo "*FAILED*"
+    fi
+    
+    rm -f $outfile
+}
+
 # This is different from $srcdir/../../bin/output_filter.sh
 STDOUT_FILTER() {
     result_file=$1
@@ -801,7 +849,6 @@ TOOLTEST h5repack_refs.h5
 # the references in attribute of compund or vlen datatype 
 TOOLTEST h5repack_attr_refs.h5 
 
-
 # Add test for memory leak in attirbute. This test is verified by CTEST.   
 # 1. leak from vlen string  
 # 2. leak from compound type without reference member
@@ -809,6 +856,10 @@ TOOLTEST h5repack_attr_refs.h5
 # Note: this test is experimental for sharing test file among tools
 TOOLTEST h5diff_attr1.h5
 
+# tests for metadata block size option ('-M')
+TOOLTEST_META h5repack_layout.h5 -M 8192
+TOOLTEST_META h5repack_layout.h5 --metadata_block_size=8192
+
 if test $nerrors -eq 0 ; then
     echo "All $TESTNAME tests passed."
     exit $EXIT_SUCCESS
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index ff682a2..e8fbbfa 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -236,109 +236,90 @@ int copy_objects(const char* fnamein,
             } /* end if */
         } /* end if */
     } /* end if */
-
-
-
-
 #if defined (H5REPACK_DEBUG_USER_BLOCK)
-    print_user_block(fnamein,fidin);
+    print_user_block(fnamein, fidin);
 #endif
 
-
     /*-------------------------------------------------------------------------
     * set the new user userblock options in the FCPL (before H5Fcreate )
     *-------------------------------------------------------------------------
     */
-
     if ( options->ublock_size > 0 )
     {
         /* either use the FCPL already created or create a new one */
-        if(fcpl != H5P_DEFAULT)
-        {
-            /* set user block size */
-            if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
-            {
-                error_msg("failed to set userblock size\n");
-                goto out;
-            }
-
-        }
-
-        else
+        if(fcpl == H5P_DEFAULT)
         {
-
             /* create a file creation property list */
             if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
             {
                 error_msg("fail to create a file creation property list\n");
                 goto out;
             }
-
-            /* set user block size */
-            if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
-            {
-                error_msg("failed to set userblock size\n");
-                goto out;
-            }
-
         }
 
-
-
+        /* set user block size */
+        if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
+        {
+            error_msg("failed to set userblock size\n");
+            goto out;
+        }
     }
 
-
     /*-------------------------------------------------------------------------
     * set alignment options
     *-------------------------------------------------------------------------
     */
-
-
     if (  options->alignment > 0 )
     {
         /* either use the FAPL already created or create a new one */
-        if (fapl != H5P_DEFAULT)
+        if (fapl == H5P_DEFAULT)
         {
-
-            if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
+            /* create a file access property list */
+            if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
             {
-                error_msg("failed to set alignment\n");
+                error_msg("Could not create file access property list\n");
                 goto out;
             }
-
         }
 
-        else
+        if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
         {
+            error_msg("failed to set alignment\n");
+            goto out;
+        }
+    }
 
+    /*-------------------------------------------------------------------------
+    * set metadata block size option
+    *-------------------------------------------------------------------------
+    */
+    if (  options->meta_block_size > 0 )
+    {
+        /* either use the FAPL already created or create a new one */
+        if (fapl == H5P_DEFAULT)
+        {
             /* create a file access property list */
             if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
             {
                 error_msg("Could not create file access property list\n");
                 goto out;
             }
-
-            if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
-            {
-                error_msg("failed to set alignment\n");
-                goto out;
-            }
-
         }
 
+        if (H5Pset_meta_block_size(fapl, options->meta_block_size) < 0)
+        {
+            error_msg("failed to set metadata block size\n");
+            goto out;
+        }
     }
 
-
     /*-------------------------------------------------------------------------
     * create the output file
     *-------------------------------------------------------------------------
     */
-
-
     if(options->verbose)
         printf("Making file <%s>...\n",fnameout);
 
-
     if((fidout = H5Fcreate(fnameout,H5F_ACC_TRUNC, fcpl, fapl)) < 0)
     {
         error_msg("<%s>: Could not create file\n", fnameout );
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index 2702ecd..2df59c3 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -35,7 +35,7 @@ const char  *outfile = NULL;
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:t:a:i:o:";
+static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:";
 static struct long_options l_opts[] = {
     { "help", no_arg, 'h' },
     { "version", no_arg, 'V' },
@@ -51,6 +51,7 @@ static struct long_options l_opts[] = {
     { "ssize", require_arg, 's' },
     { "ublock", require_arg, 'u' },
     { "block", require_arg, 'b' },
+    { "metadata_block_size", require_arg, 'M' },
     { "threshold", require_arg, 't' },
     { "alignment", require_arg, 'a' },
     { "infile", require_arg, 'i' },   /* -i for backward compability */
@@ -76,24 +77,6 @@ static struct long_options l_opts[] = {
  *
  * Comments:
  *
- * Modifications:
- *  July 2004: Introduced the extra EC or NN option for SZIP
- *  October 2006: Added a new switch -n, that allows to write the dataset
- *                using a native type. The default to write is the file type.
- *
- * Modification:
- *   Peter Cao, June 13, 2007
- *    Add "-L, --latest" option to pack a file with the latest file format
- *   PVN, November 19, 2007
- *    adopted the syntax h5repack [OPTIONS]  file1 file2
- *   PVN, November 28, 2007
- *    added support for multiple global filters
- *   PVN, May 16, 2008
- *    added  backward compatibility for -i infile -o outfile
- *   PVN, August 20, 2008
- *    add a user block to repacked file (switches -u -b)
- *   PVN, August 28, 2008
- *    add options to set alignment (H5Pset_alignment) (switches -t -a)
  *-------------------------------------------------------------------------
  */
 int main(int argc, const char **argv)
@@ -184,6 +167,7 @@ static void usage(const char *prog)
 
  printf("   -u U, --ublock=U        Name of file U with user block data to be added\n");
  printf("   -b B, --block=B         Size of user block to be added\n");
+ printf("   -M A, --metadata_block_size=A  Metadata block size for H5Pset_meta_block_size\n");
  printf("   -t T, --threshold=T     Threshold value for H5Pset_alignment\n");
  printf("   -a A, --alignment=A     Alignment value for H5Pset_alignment\n");
 
@@ -412,7 +396,6 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options)
 
             break;
 
-
         case 'u':
 
             options->ublock_filename = opt_arg;
@@ -423,6 +406,11 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options)
             options->ublock_size = (hsize_t)HDatol( opt_arg );
             break;
 
+        case 'M':
+
+            options->meta_block_size = (hsize_t)HDatol( opt_arg );
+            break;
+
         case 't':
 
             options->threshold = (hsize_t)HDatol( opt_arg );
@@ -437,7 +425,6 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options)
                 HDexit(EXIT_FAILURE);
             }
             break;
-
         } /* switch */
 
 
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index 95b90d4..2efe29a 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -78,7 +78,6 @@
 #define FNAME17OUT  "h5repack_named_dtypes_out.h5"
 
 #define FNAME18     "h5repack_layout2.h5"
-#define FNAME18OUT  "h5repack_layout2_out.h5"
 
 #define FNAME_UB   "ublock.bin"
 
@@ -1548,6 +1547,25 @@ int main (void)
     PASSED();
 
     /*-------------------------------------------------------------------------
+    * test --metadata_block_size option
+    *-------------------------------------------------------------------------
+    */
+    TESTING("    metadata block size option");
+    if (h5repack_init (&pack_options, 0) < 0)
+        GOERROR;
+    pack_options.meta_block_size = 8192;
+    if(h5repack(FNAME1, FNAME1OUT, &pack_options) < 0)
+        GOERROR;
+    if(h5diff(FNAME1, FNAME1OUT, NULL, NULL, &diff_options) > 0)
+        GOERROR;
+    if(h5repack_verify(FNAME1OUT, &pack_options) <= 0)
+        GOERROR;
+    if(h5repack_end(&pack_options) < 0)
+        GOERROR;
+    PASSED();
+
+
+    /*-------------------------------------------------------------------------
     * clean temporary test files
     *-------------------------------------------------------------------------
     */
@@ -3229,7 +3247,7 @@ make_userblock(void)
 
     /* Initialize userblock data */
     for(u = 0; u < USERBLOCK_SIZE; u++)
-        ub[u] = 'a' + (u % 26);
+        ub[u] = 'a' + (char)(u % 26);
 
     /* Re-open HDF5 file, as "plain" file */
     if((fd = HDopen(FNAME16, O_WRONLY, 0644)) < 0)
@@ -3342,7 +3360,7 @@ make_userblock_file(void)
 
     /* initialize userblock data */
     for(u = 0; u < USERBLOCK_SIZE; u++)
-        ub[u] = 'a' + (u % 26);
+        ub[u] = 'a' + (char)(u % 26);
 
     /* open file */
     if((fd = HDopen(FNAME_UB,O_WRONLY|O_CREAT|O_TRUNC, 0644 )) < 0)
@@ -3818,7 +3836,7 @@ int write_dset_in(hid_t loc_id,
             int l;
 
             buf52[i][j].p = malloc((i + 1) * sizeof(int));
-            buf52[i][j].len = i + 1;
+            buf52[i][j].len = (size_t)(i + 1);
             for(l = 0; l < i + 1; l++)
             {
                 if(make_diffs)
@@ -4039,7 +4057,7 @@ int write_dset_in(hid_t loc_id,
                 int l;
 
                 buf53[i][j][k].p = malloc((i + 1) * sizeof(int));
-                buf53[i][j][k].len = i + 1;
+                buf53[i][j][k].len = (size_t)(i + 1);
                 for(l = 0; l < i + 1; l++)
                 {
                     if(make_diffs)
@@ -4830,7 +4848,7 @@ int write_attr_in(hid_t loc_id,
         {
             int l;
             buf52[i][j].p = malloc((i + 1) * sizeof(int));
-            buf52[i][j].len = i + 1;
+            buf52[i][j].len = (size_t)(i + 1);
             for (l = 0; l < i + 1; l++)
                 if (make_diffs)((int *)buf52[i][j].p)[l] = 0;
                 else ((int *)buf52[i][j].p)[l] = n++;
@@ -5296,7 +5314,7 @@ int write_attr_in(hid_t loc_id,
             {
                 int l;
                 buf53[i][j][k].p = malloc((i + 1) * sizeof(int));
-                buf53[i][j][k].len = i + 1;
+                buf53[i][j][k].len = (size_t)i + 1;
                 for (l = 0; l < i + 1; l++)
                     if (make_diffs)
                     {