[svn-r19076] Purpose: Fix bug 1951

Description:
A bug introduced in 1.8.5 causes local heap data blocks to be mis-aligned when
sizeof_offsets + 2*sizeof_lengths is not a multiple of 8.  In this case, the
address of the data block as stored in the heap prefix is aligned but the actual
data block is not.  This causes files created with these sizes to be corrupted,
and prevents uncorrupted files with these sizes to be unreadable.  Modified
local heap code to account for alignment.

Tested: jam, amani, linew (h5committest)

9 files changed, 150 insertions, 16 deletions

diff --git a/MANIFEST b/MANIFEST
index defd2b0..d597d53 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -856,6 +856,7 @@
 ./test/gen_old_group.c			_DO_NOT_DISTRIBUTE_
 ./test/gen_old_layout.c			_DO_NOT_DISTRIBUTE_
 ./test/gen_old_mtime.c			_DO_NOT_DISTRIBUTE_
+./test/gen_sizes_lheap.c                _DO_NOT_DISTRIBUTE_
 ./test/gen_specmetaread.c		_DO_NOT_DISTRIBUTE_
 ./test/gen_udlinks.c			_DO_NOT_DISTRIBUTE_
 ./test/getname.c
@@ -917,6 +918,7 @@
 ./test/trefer.c
 ./test/trefstr.c
 ./test/tselect.c
+./test/tsizeslheap.h5
 ./test/tskiplist.c
 ./test/tsohm.c
 ./test/ttst.c
diff --git a/configure b/configure
index 66f8fa7..920148e 100755
--- a/configure
+++ b/configure
@@ -1,5 +1,5 @@
 #! /bin/sh
-# From configure.in Id: configure.in 19064 2010-07-11 17:06:05Z hdftest .
+# From configure.in Id: configure.in 19068 2010-07-14 15:56:42Z acheng .
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.65 for HDF5 1.9.74.
 #
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 6af47e9..02ff846 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -234,6 +234,10 @@ Bug Fixes since HDF5-1.8.0 release
 
     Library
     -------
+      - Fixed a bug that could cause file corruption when using non-default
+            sizes of addresses and/or lengths.  This bug could also cause
+            uncorrupted files with this property to be unreadable.  This bug
+            was introduced in 1.8.5. (NAF - 2010/07/16 - 1951)
       - Fixed a file corruption bug that could happen when shrinking a
             compressed dataset. (NAF - 2010/05/20)
       - Fixed some memory leaks in VL datatype conversion when strings are
diff --git a/src/H5HLcache.c b/src/H5HLcache.c
index 238dce6..51d60f9 100644
--- a/src/H5HLcache.c
+++ b/src/H5HLcache.c
@@ -330,6 +330,11 @@ H5HL_prefix_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)
 
             /* Check if the current buffer from the speculative read already has the heap data */
             if(spec_read_size >= (heap->prfx_size + heap->dblk_size)) {
+                /* Set p to the start of the data block.  This is necessary
+                 * because there may be a gap between the used portion of the
+                 * prefix and the data block due to alignment constraints. */
+                p = buf + heap->prfx_size;
+
                 /* Copy the heap data from the speculative read buffer */
                 HDmemcpy(heap->dblk_image, p, heap->dblk_size);
             } /* end if */
@@ -435,6 +440,11 @@ H5HL_prefix_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
 
         /* Check if the local heap is a single object in cache */
         if(heap->single_cache_obj) {
+            /* Set p to the start of the data block.  This is necessary because
+             * there may be a gap between the used portion of the prefix and the
+             * data block due to alignment constraints. */
+            p = buf + heap->prfx_size;
+
             /* Serialize the free list into the heap data's image */
             H5HL_fl_serialize(heap);
 
diff --git a/test/Makefile.am b/test/Makefile.am
index 043e05d..07df450 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -62,7 +62,8 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version testmeta
 # so do not appear in this list.
 BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \
     gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \
-    gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread
+    gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \
+    gen_sizes_lheap
 
 if BUILD_ALL_CONDITIONAL
   noinst_PROGRAMS=$(BUILD_ALL_PROGS)
diff --git a/test/Makefile.in b/test/Makefile.in
index f520ed1..e441918 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -96,7 +96,7 @@ am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \
 	gen_new_super$(EXEEXT) gen_noencoder$(EXEEXT) \
 	gen_nullspace$(EXEEXT) gen_udlinks$(EXEEXT) \
 	space_overflow$(EXEEXT) gen_filespace$(EXEEXT) \
-	gen_specmetaread$(EXEEXT)
+	gen_specmetaread$(EXEEXT) gen_sizes_lheap$(EXEEXT)
 PROGRAMS = $(noinst_PROGRAMS)
 app_ref_SOURCES = app_ref.c
 app_ref_OBJECTS = app_ref.$(OBJEXT)
@@ -254,6 +254,10 @@ gen_nullspace_SOURCES = gen_nullspace.c
 gen_nullspace_OBJECTS = gen_nullspace.$(OBJEXT)
 gen_nullspace_LDADD = $(LDADD)
 gen_nullspace_DEPENDENCIES = libh5test.la $(LIBHDF5)
+gen_sizes_lheap_SOURCES = gen_sizes_lheap.c
+gen_sizes_lheap_OBJECTS = gen_sizes_lheap.$(OBJEXT)
+gen_sizes_lheap_LDADD = $(LDADD)
+gen_sizes_lheap_DEPENDENCIES = libh5test.la $(LIBHDF5)
 gen_specmetaread_SOURCES = gen_specmetaread.c
 gen_specmetaread_OBJECTS = gen_specmetaread.$(OBJEXT)
 gen_specmetaread_LDADD = $(LDADD)
@@ -385,12 +389,12 @@ SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c btree2.c \
 	gen_bogus.c gen_cross.c gen_deflate.c gen_filespace.c \
 	gen_filters.c gen_new_array.c gen_new_fill.c gen_new_group.c \
 	gen_new_mtime.c gen_new_super.c gen_noencoder.c \
-	gen_nullspace.c gen_specmetaread.c gen_udlinks.c getname.c \
-	gheap.c hyperslab.c istore.c lheap.c links.c mf.c mount.c \
-	mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \
-	set_extent.c space_overflow.c stab.c tcheck_version.c \
-	$(testhdf5_SOURCES) testmeta.c $(ttsafe_SOURCES) unlink.c \
-	vfd.c
+	gen_nullspace.c gen_sizes_lheap.c gen_specmetaread.c \
+	gen_udlinks.c getname.c gheap.c hyperslab.c istore.c lheap.c \
+	links.c mf.c mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c \
+	reserved.c set_extent.c space_overflow.c stab.c \
+	tcheck_version.c $(testhdf5_SOURCES) testmeta.c \
+	$(ttsafe_SOURCES) unlink.c vfd.c
 DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \
 	btree2.c cache.c cache_api.c cache_tagging.c cmpd_dset.c \
 	cross_read.c dangle.c dsets.c dt_arith.c dtransform.c dtypes.c \
@@ -399,12 +403,12 @@ DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \
 	gen_bad_ohdr.c gen_bogus.c gen_cross.c gen_deflate.c \
 	gen_filespace.c gen_filters.c gen_new_array.c gen_new_fill.c \
 	gen_new_group.c gen_new_mtime.c gen_new_super.c \
-	gen_noencoder.c gen_nullspace.c gen_specmetaread.c \
-	gen_udlinks.c getname.c gheap.c hyperslab.c istore.c lheap.c \
-	links.c mf.c mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c \
-	reserved.c set_extent.c space_overflow.c stab.c \
-	tcheck_version.c $(testhdf5_SOURCES) testmeta.c \
-	$(ttsafe_SOURCES) unlink.c vfd.c
+	gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \
+	gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \
+	istore.c lheap.c links.c mf.c mount.c mtime.c ntypes.c \
+	objcopy.c ohdr.c pool.c reserved.c set_extent.c \
+	space_overflow.c stab.c tcheck_version.c $(testhdf5_SOURCES) \
+	testmeta.c $(ttsafe_SOURCES) unlink.c vfd.c
 ETAGS = etags
 CTAGS = ctags
 am__tty_colors = \
@@ -729,7 +733,8 @@ TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \
 # so do not appear in this list.
 BUILD_ALL_PROGS = gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \
     gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \
-    gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread
+    gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \
+    gen_sizes_lheap
 
 
 # The libh5test library provides common support code for the tests.
@@ -959,6 +964,9 @@ gen_noencoder$(EXEEXT): $(gen_noencoder_OBJECTS) $(gen_noencoder_DEPENDENCIES)
 gen_nullspace$(EXEEXT): $(gen_nullspace_OBJECTS) $(gen_nullspace_DEPENDENCIES) 
 	@rm -f gen_nullspace$(EXEEXT)
 	$(LINK) $(gen_nullspace_OBJECTS) $(gen_nullspace_LDADD) $(LIBS)
+gen_sizes_lheap$(EXEEXT): $(gen_sizes_lheap_OBJECTS) $(gen_sizes_lheap_DEPENDENCIES) 
+	@rm -f gen_sizes_lheap$(EXEEXT)
+	$(LINK) $(gen_sizes_lheap_OBJECTS) $(gen_sizes_lheap_LDADD) $(LIBS)
 gen_specmetaread$(EXEEXT): $(gen_specmetaread_OBJECTS) $(gen_specmetaread_DEPENDENCIES) 
 	@rm -f gen_specmetaread$(EXEEXT)
 	$(LINK) $(gen_specmetaread_OBJECTS) $(gen_specmetaread_LDADD) $(LIBS)
@@ -1081,6 +1089,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_new_super.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_noencoder.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_nullspace.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_sizes_lheap.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_specmetaread.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_udlinks.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getname.Po@am__quote@
diff --git a/test/gen_sizes_lheap.c b/test/gen_sizes_lheap.c
new file mode 100644
index 0000000..de7e261
--- /dev/null
+++ b/test/gen_sizes_lheap.c
@@ -0,0 +1,82 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the files COPYING and Copyright.html.  COPYING can be found at the root   *
+ * of the source code distribution tree; Copyright.html can be found at the  *
+ * root level of an installed copy of the electronic HDF5 document set and   *
+ * is linked from the top-level documents page.  It can also be found at     *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have          *
+ * access to either file, you may request a copy from help@hdfgroup.org.     *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer:  Neil Fortner <nfortne2@hdfgroup.org>
+ *              Thursday, July  15, 2010
+ *
+ * Purpose:     Creates a file with non-default sizes of lengths and addresses.
+ *              This is used to make sure that the local heap code is able to
+ *              handle this case correctly, even when the heap prefix and data
+ *              are contiguous.
+ */
+
+#include "hdf5.h"
+
+#define TESTFILE   "tsizeslheap.h5"
+
+
+/*-------------------------------------------------------------------------
+ * Function:    main
+ *
+ * Purpose:
+ *
+ * Return:      Success:
+ *
+ *              Failure:
+ *
+ * Programmer:  Neil Fortner
+ *              Thursday, July  15, 2010
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+    hid_t       file, space, dset, fcpl;
+
+    /* Create the FCPL */
+    fcpl = H5Pcreate(H5P_FILE_CREATE);
+    if(fcpl < 0)
+        printf("fcpl < 0!\n");
+
+    /* Set sizeof_addr and sizeof_size to be 4 */
+    if(H5Pset_sizes(fcpl, 4, 4) < 0)
+        printf("H5Pset_sizes < 0!\n");
+
+    /* Create the file */
+    file = H5Fcreate(TESTFILE, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT);
+    if(file < 0)
+        printf("file < 0!\n");
+
+    /* Create the dataspace (for dataset) */
+    space = H5Screate(H5S_SCALAR);
+    if(space < 0)
+        printf("space < 0!\n");
+
+    /* Create the dataset with compound array fields */
+    dset = H5Dcreate2(file, "Dataset1", H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    if(dset < 0)
+        printf("dset < 0!\n");
+
+    H5Dclose(dset);
+    H5Sclose(space);
+    H5Fclose(file);
+    H5Pclose(fcpl);
+
+    return 0;
+}
diff --git a/test/lheap.c b/test/lheap.c
index 1d98fc4..9f78a15 100644
--- a/test/lheap.c
+++ b/test/lheap.c
@@ -20,6 +20,7 @@
  * Purpose:	Test local heaps used by symbol tables (groups).
  */
 #include "h5test.h"
+#include "H5srcdir.h"
 #include "H5ACprivate.h"
 #include "H5HLprivate.h"
 #include "H5Iprivate.h"
@@ -29,6 +30,8 @@ const char *FILENAME[] = {
     NULL
 };
 
+#define TESTFILE "tsizeslheap.h5"
+
 #define NOBJS   40
 
 
@@ -170,6 +173,29 @@ main(void)
 
     if (H5Fclose(file)<0) goto error;
     PASSED();
+
+    /* Check opening existing file non-default sizes of lengths and addresses */
+    TESTING("opening pre-created file with non-default sizes");
+    {
+        const char *testfile = H5_get_srcdir_filename(TESTFILE); /* Corrected test file name */
+        hid_t dset = -1;
+
+        file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT);
+        if(file >= 0){
+            if((dset = H5Dopen2(file, "/Dataset1", H5P_DEFAULT)) < 0)
+                TEST_ERROR
+            if(H5Dclose(dset) < 0) TEST_ERROR
+            if(H5Fclose(file) < 0) TEST_ERROR
+        }
+        else {
+            H5_FAILED();
+            printf("***cannot open the pre-created non-default sizes test file (%s)\n",
+                testfile);
+            goto error;
+        } /* end else */
+    }
+    PASSED();
+
     puts("All local heap tests passed.");
     h5_cleanup(FILENAME, fapl);
 
diff --git a/test/tsizeslheap.h5 b/test/tsizeslheap.h5
new file mode 100644
index 0000000..c7f2adb
--- /dev/null
+++ b/test/tsizeslheap.h5