[svn-r22127] Purpose: Fix earray failure

Description:
In H5Dearray.c, functions would "swizzle" the chunk offset and "down" number of
chunks in order to more the unlimited dimension to be the first dimension, but
they would not swizzle the chunk dimensions.  This could cause two chunks to
have the same index, causing problems.  Modified code to swizzle the chunk
dimensions, and added a test.

Note: There is still a problem with h5watch that appears to be unrelated.

Tested: durandal

5 files changed, 168 insertions, 87 deletions

diff --git a/src/H5Dearray.c b/src/H5Dearray.c
index 72e07e0..40c5c5f 100644
--- a/src/H5Dearray.c
+++ b/src/H5Dearray.c
@@ -1104,10 +1104,10 @@ H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
 
         /* Set up the swizzled chunk coordinates */
         HDmemcpy(swizzled_coords, udata->common.offset, ndims * sizeof(udata->common.offset[0]));
-        H5V_swizzle_coords(swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_coords, idx_info->layout->u.earray.unlim_dim);
 
         /* Calculate the index of this chunk */
-        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
+        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->u.earray.swizzled_dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
     } /* end if */
     else {
@@ -1270,10 +1270,10 @@ H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
 
         /* Set up the swizzled chunk coordinates */
         HDmemcpy(swizzled_coords, udata->common.offset, ndims * sizeof(udata->common.offset[0]));
-        H5V_swizzle_coords(swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_coords, idx_info->layout->u.earray.unlim_dim);
 
         /* Calculate the index of this chunk */
-        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
+        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->u.earray.swizzled_dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
     } /* end if */
     else {
@@ -1333,12 +1333,18 @@ H5D_earray_idx_resize(H5O_layout_chunk_t *layout)
     /* Check args */
     HDassert(layout);
 
-    /* Set up the swizzled "down" chunk information */
+
+    /* "Swizzle" constant dimensions for this dataset */
     if(layout->u.earray.unlim_dim > 0) {
         hsize_t swizzled_chunks[H5O_LAYOUT_NDIMS];    /* Swizzled form of # of chunks in each dimension */
 
+        /* Get the swizzled chunk dimensions */
+        HDmemcpy(layout->u.earray.swizzled_dim, layout->dim, (layout->ndims - 1) * sizeof(layout->dim[0]));
+        H5V_swizzle_coords(uint32_t, layout->u.earray.swizzled_dim, layout->u.earray.unlim_dim);
+
+        /* Get the swizzled number of chunks in each dimension */
         HDmemcpy(swizzled_chunks, layout->chunks, (layout->ndims - 1) * sizeof(swizzled_chunks[0]));
-        H5V_swizzle_coords(swizzled_chunks, layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_chunks, layout->u.earray.unlim_dim);
 
         /* Get the swizzled "down" sizes for each dimension */
         if(H5V_array_down((layout->ndims - 1), swizzled_chunks, layout->u.earray.swizzled_down_chunks) < 0)
@@ -1511,10 +1517,11 @@ H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
 
         /* Set up the swizzled chunk coordinates */
         HDmemcpy(swizzled_coords, udata->offset, ndims * sizeof(udata->offset[0]));
-        H5V_swizzle_coords(swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+
 
         /* Calculate the index of this chunk */
-        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
+        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->u.earray.swizzled_dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
     } /* end if */
     else {
@@ -1911,10 +1918,10 @@ H5D_earray_idx_support(const H5D_chk_idx_info_t *idx_info,
 
         /* Set up the swizzled chunk coordinates */
         HDmemcpy(swizzled_coords, udata->offset, ndims * sizeof(udata->offset[0]));
-        H5V_swizzle_coords(swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_coords, idx_info->layout->u.earray.unlim_dim);
 
         /* Calculate the index of this chunk */
-        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
+        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->u.earray.swizzled_dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
     } /* end if */
     else {
@@ -1981,10 +1988,11 @@ H5D_earray_idx_unsupport(const H5D_chk_idx_info_t *idx_info,
 
         /* Set up the swizzled chunk coordinates */
         HDmemcpy(swizzled_coords, udata->offset, ndims * sizeof(udata->offset[0]));
-        H5V_swizzle_coords(swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+        H5V_swizzle_coords(hsize_t, swizzled_coords, idx_info->layout->u.earray.unlim_dim);
+
 
         /* Calculate the index of this chunk */
-        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
+        if(H5V_chunk_index(ndims, swizzled_coords, idx_info->layout->u.earray.swizzled_dim, idx_info->layout->u.earray.swizzled_down_chunks, &idx) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
     } /* end if */
     else {
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index c05fcb4..c5074f9 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -467,6 +467,7 @@ typedef struct H5O_layout_chunk_earray_t {
     } cparam;
 
     unsigned    unlim_dim;              /* Rank of unlimited dimension for dataset */
+    uint32_t    swizzled_dim[H5O_LAYOUT_NDIMS]; /* swizzled chunk dimensions */
     hsize_t    	swizzled_down_chunks[H5O_LAYOUT_NDIMS];	/* swizzled "down" size of number of chunks in each dimension */
 } H5O_layout_chunk_earray_t;
 
diff --git a/src/H5V.c b/src/H5V.c
index fe38978..e23af01 100644
--- a/src/H5V.c
+++ b/src/H5V.c
@@ -1295,75 +1295,6 @@ H5V_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk,
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5V_swizzle_coords
- *
- * Purpose:	Given a coordinate offset array (COORDS), move the unlimited
- *		dimension (UNLIM_DIM) value to offset 0, sliding any
- *		intermediate values down one position.
- *
- * Return:	None
- *
- * Programmer:	Quincey Koziol
- *		Tuesday, July 21, 2009
- *
- *-------------------------------------------------------------------------
- */
-void
-H5V_swizzle_coords(hsize_t *coords, unsigned unlim_dim)
-{
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
-
-    /* Sanity check */
-    HDassert(coords);
-
-    /* Nothing to do when unlimited dimension is at position 0 */
-    if(0 != unlim_dim) {
-        hsize_t tmp = coords[unlim_dim];
-
-        HDmemmove(&coords[1], &coords[0], sizeof(coords[0]) * unlim_dim);
-        coords[0] = tmp;
-    } /* end if */
-
-    FUNC_LEAVE_NOAPI_VOID
-} /* end H5V_swizzle_coords() */
-
-
-/*-------------------------------------------------------------------------
- * Function:	H5V_unswizzle_coords
- *
- * Purpose:	Given a coordinate offset array (COORDS), move the value at
- *		offset 0 to offset of the unlimied dimension (UNLIM_DIM),
- *		sliding any intermediate values up one position.  Undoes the
- *		"swizzle_coords" operation.
- *
- * Return:	None
- *
- * Programmer:	Quincey Koziol
- *		Tuesday, July 21, 2009
- *
- *-------------------------------------------------------------------------
- */
-void
-H5V_unswizzle_coords(hsize_t *coords, unsigned unlim_dim)
-{
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
-
-    /* Sanity check */
-    HDassert(coords);
-
-    /* Nothing to do when unlimited dimension is at position 0 */
-    if(0 != unlim_dim) {
-        hsize_t tmp = coords[0];
-
-        HDmemmove(&coords[0], &coords[1], sizeof(coords[0]) * unlim_dim);
-        coords[unlim_dim] = tmp;
-    } /* end if */
-
-    FUNC_LEAVE_NOAPI_VOID
-} /* end H5V_unswizzle_coords() */
-
-
-/*-------------------------------------------------------------------------
  * Function:	H5V_opvv
  *
  * Purpose:	Perform an operation on a source & destination sequences
diff --git a/src/H5Vprivate.h b/src/H5Vprivate.h
index 89e1938..c2d4e4a 100644
--- a/src/H5Vprivate.h
+++ b/src/H5Vprivate.h
@@ -49,6 +49,39 @@ typedef herr_t (*H5V_opvv_func_t)(hsize_t dst_off, hsize_t src_off,
 
 #define H5V_vector_zero(N,DST) HDmemset(DST,0,(N)*sizeof(*(DST)))
 
+/* Given a coordinate offset array (COORDS) of type TYPE, move the unlimited
+ * dimension (UNLIM_DIM) value to offset 0, sliding any intermediate values down
+ * one position. */
+#define H5V_swizzle_coords(TYPE,COORDS,UNLIM_DIM) {                            \
+    /* COORDS must be an array of type TYPE */                                 \
+    HDassert(sizeof(COORDS[0]) == sizeof(TYPE));                               \
+                                                                               \
+    /* Nothing to do when unlimited dimension is at position 0 */              \
+    if(0 != (UNLIM_DIM)) {                                                     \
+        TYPE _tmp = (COORDS)[UNLIM_DIM];                                       \
+                                                                               \
+        HDmemmove(&(COORDS)[1], &(COORDS)[0], sizeof(TYPE) * (UNLIM_DIM));     \
+        (COORDS)[0] = _tmp;                                                    \
+    } /* end if */                                                             \
+}
+
+/* Given a coordinate offset array (COORDS) of type TYPE, move the value at
+ * offset 0 to offset of the unlimied dimension (UNLIM_DIM), sliding any
+ * intermediate values up one position.  Undoes the "swizzle_coords" operation.
+ */
+#define H5V_unswizzle_coords(TYPE,COORDS,UNLIM_DIM) {                          \
+    /* COORDS must be an array of type TYPE */                                 \
+    HDassert(sizeof(COORDS[0]) == sizeof(TYPE));                               \
+                                                                               \
+    /* Nothing to do when unlimited dimension is at position 0 */              \
+    if(0 != (UNLIM_DIM)) {                                                     \
+        TYPE _tmp = (COORDS)[0];                                               \
+                                                                               \
+        HDmemmove(&(COORDS)[0], &(COORDS)[1], sizeof(TYPE) * (UNLIM_DIM));     \
+        (COORDS)[UNLIM_DIM] = _tmp;                                            \
+    } /* end if */                                                             \
+}
+
 /* A null pointer is equivalent to a zero vector */
 #define H5V_ZERO        NULL
 
@@ -93,8 +126,6 @@ H5_DLL herr_t H5V_array_calc(hsize_t offset, unsigned n,
     const hsize_t *total_size, hsize_t *coords);
 H5_DLL herr_t H5V_chunk_index(unsigned ndims, const hsize_t *coord,
     const uint32_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx);
-H5_DLL void H5V_swizzle_coords(hsize_t *coords, unsigned unlim_dim);
-H5_DLL void H5V_unswizzle_coords(hsize_t *coords, unsigned unlim_dim);
 H5_DLL ssize_t H5V_opvv(size_t dst_max_nseq, size_t *dst_curr_seq, size_t dst_len_arr[],
     hsize_t dst_off_arr[],
     size_t src_max_nseq, size_t *src_curr_seq, size_t src_len_arr[],
diff --git a/test/dsets.c b/test/dsets.c
index a397150..898782d 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -8410,7 +8410,7 @@ test_chunk_fast(hid_t fapl)
                             if(H5V_array_calc_pre(u, ndims, down, hs_offset) < 0) FAIL_STACK_ERROR
 
                             /* Un-swizzle hyperslab offset in same way as swizzled dimensions */
-                            H5V_unswizzle_coords(hs_offset, unlim_dim);
+                            H5V_unswizzle_coords(hsize_t, hs_offset, unlim_dim);
 
                             /* Select a single element in the dataset */
                             if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) FAIL_STACK_ERROR
@@ -8460,7 +8460,7 @@ test_chunk_fast(hid_t fapl)
                                 if(H5V_array_calc(u, ndims, swizzled_dim, hs_offset) < 0) FAIL_STACK_ERROR
 
                                 /* Un-swizzle hyperslab offset in same way as swizzled dimensions */
-                                H5V_unswizzle_coords(hs_offset, unlim_dim);
+                                H5V_unswizzle_coords(hsize_t, hs_offset, unlim_dim);
 
                                 /* Select a single element in the dataset */
                                 if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) FAIL_STACK_ERROR
@@ -8524,7 +8524,7 @@ test_chunk_fast(hid_t fapl)
                         if(H5Sget_simple_extent_dims(sid, swizzled_dim, NULL) < 0) FAIL_STACK_ERROR
 
                         /* Generate the swizzled dimensions */
-                        H5V_swizzle_coords(swizzled_dim, unlim_dim);
+                        H5V_swizzle_coords(hsize_t, swizzled_dim, unlim_dim);
 
                         /* Compute the "down" dimension values */
                         if(H5V_array_down(ndims, swizzled_dim, down) < 0) FAIL_STACK_ERROR
@@ -8535,7 +8535,7 @@ test_chunk_fast(hid_t fapl)
                             if(H5V_array_calc_pre(u, ndims, down, hs_offset) < 0) FAIL_STACK_ERROR
 
                             /* Unswizzle hyperslab offset in same way as swizzled dimensions */
-                            H5V_unswizzle_coords(hs_offset, unlim_dim);
+                            H5V_unswizzle_coords(hsize_t, hs_offset, unlim_dim);
 
                             /* Select a single element in the dataset */
                             if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) FAIL_STACK_ERROR
@@ -8592,6 +8592,7 @@ error:
     return -1;
 } /* end test_chunk_fast() */
 
+
 /*-------------------------------------------------------------------------
  * Function: test_reopen_chunk_fast
  *
@@ -8714,6 +8715,114 @@ error:
     return -1;
 } /* end test_reopen_chunk_fast() */
 
+
+/*-------------------------------------------------------------------------
+ * Function: test_chunk_fast_bug1
+ *
+ * Purpose:     Test extensible arrays where the first dimension in the
+ *              chunk size is the same as the second dimension in the
+ *              dataset size.  This helps to confirm that all dimensions
+ *              are being "swizzled" correctly in the earray chunk index
+ *              code.
+ *
+ * Return:      Success: 0
+ *              Failure: -1
+ *
+ * Programmer:  Neil Fortner
+ *              March 22, 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_chunk_fast_bug1(hid_t fapl)
+{
+    char        filename[FILENAME_BUF_SIZE];
+    hid_t       fid = -1;       /* File ID */
+    hid_t       dcpl = -1;      /* Dataset creation property list ID */
+    hid_t       sid = -1;       /* Dataspace ID */
+    hid_t       dsid = -1;      /* Dataset ID */
+    hsize_t     dim[2], max_dim[2], chunk_dim[2]; /* Dataset and chunk dimensions */
+    H5D_alloc_time_t alloc_time;        /* Storage allocation time */
+    static unsigned wbuf[40][20], rbuf[40][20];  /* Element written/read */
+    unsigned    i, j;            /* Local index variables */
+
+    TESTING("datasets w/extensible array chunk indexing bug");
+
+    h5_fixname(FILENAME[10], fapl, filename, sizeof filename);
+
+    /* Initialize write buffer */
+    for(i=0; i<40; i++)
+        for(j=0; j<20; j++)
+            wbuf[i][j] = (i * 20) + j;
+
+    /* Create 2-D dataspace */
+    dim[0] = 40;
+    dim[1] = 20;
+    max_dim[0] = 40;
+    max_dim[1] = H5S_UNLIMITED;
+    if((sid = H5Screate_simple(2, dim, max_dim)) < 0) FAIL_STACK_ERROR
+
+    /* Loop over storage allocation time */
+    for(alloc_time = H5D_ALLOC_TIME_EARLY; alloc_time <= H5D_ALLOC_TIME_INCR; alloc_time++) {
+        /* Create file */
+        if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR
+
+        /* Create dataset creation property list */
+        if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR
+
+        /* Set chunking */
+        chunk_dim[0] = 20;
+        chunk_dim[1] = 10;
+        if(H5Pset_chunk(dcpl, 2, chunk_dim) < 0) FAIL_STACK_ERROR
+
+        /* Set allocation time */
+        if(H5Pset_alloc_time(dcpl, alloc_time) < 0) FAIL_STACK_ERROR
+
+        /* Create chunked dataset */
+        if((dsid = H5Dcreate2(fid, "dset", H5T_NATIVE_UINT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
+            FAIL_STACK_ERROR
+
+        /* Write buffer to dataset */
+        if(H5Dwrite(dsid, H5T_NATIVE_UINT, sid, sid, H5P_DEFAULT, &wbuf) < 0)
+            FAIL_STACK_ERROR
+
+        /* Close everything */
+        if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR
+
+        /* Reopen the dataset */
+        if((dsid = H5Dopen2(fid, "dset", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+
+        /* Read from dataset */
+        if(H5Dread(dsid, H5T_NATIVE_UINT, sid, sid, H5P_DEFAULT, &rbuf) < 0)
+            FAIL_STACK_ERROR
+
+        /* Verify read data */
+        for(i=0; i<40; i++)
+            for(j=0; j<20; j++)
+                if(wbuf[i][j] != rbuf[i][j])
+                    FAIL_PUTS_ERROR("invalid element read");
+
+        if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR
+        if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR
+        if(H5Fclose(fid) < 0) FAIL_STACK_ERROR
+
+    } /* end for */
+
+    if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+
+    PASSED();
+    return 0;
+
+error:
+    H5E_BEGIN_TRY {
+        H5Pclose(dcpl);
+        H5Dclose(dsid);
+        H5Sclose(sid);
+        H5Fclose(fid);
+    } H5E_END_TRY;
+    return -1;
+} /* end test_chunk_fast_bug1() */
+
 /* This message derives from H5Z */
 const H5Z_class2_t H5Z_EXPAND[1] = {{
     H5Z_CLASS_T_VERS,           /* H5Z_class_t version */
@@ -10021,6 +10130,7 @@ main(void)
         nerrors += (test_big_chunks_bypass_cache(my_fapl) < 0   ? 1 : 0);
         nerrors += (test_chunk_fast(my_fapl) < 0		? 1 : 0);
         nerrors += (test_reopen_chunk_fast(my_fapl) < 0		? 1 : 0);
+        nerrors += (test_chunk_fast_bug1(my_fapl) < 0           ? 1 : 0);
         nerrors += (test_chunk_expand(my_fapl) < 0		? 1 : 0);
 	nerrors += (test_layout_extend(my_fapl) < 0		? 1 : 0);
         nerrors += (test_fixed_array(my_fapl) < 0		? 1 : 0);