[svn-r24278] Purpose: Block v-1 B-tree writes under SWMR write semantics

Description:
Modified the MD cache so that v-1 B-tree node writes are prohibited
under SWMR semantics.

Testing changes:
testswmr
Removed the -i b1 option from the test.
NOTE: The testswmr.sh test performs operations that are not currently
supported under our current level of SWMR implementation.

links
Modified an external link test so that a close/reopen w/ SWMR write
test is not performed when the v-1 B-trees are used.

dsets
Chunk tests don't run when using SWMR and the old file formt (i.e.
v-1 B-tree chunk indexes).  Also added a new test that fails when
trying to write v-1 B-tree nodes. Makefile updated to clean up
all test files created by this test.

flushrefresh
Modified the test to use the latest file format.

Tested on:
    jam         (32-bit LE linux)
    koala       (64-bit LE linux)
    ostrich     (64-bit BE linux)

7 files changed, 176 insertions, 47 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index e81f487..e1c152c 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -134,10 +134,10 @@ flush2.chkexe_: flush1.chkexe_
 # prefix or low-level driver with environment variables will influence
 # the temporary file name in ways that the makefile is not aware of.
 CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 \
-    dset_offset.h5 \
+    dset_offset.h5 chunk_fixed.h5 \
     max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \
     huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_fast.h5 \
-    chunk_expand.h5 \
+    chunk_expand.h5 layout_extend.h5 swmr_fail.h5 partial_chunks.h5 \
     copy_dcpl_newfile.h5 extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \
     sys_file1 tfile[1-6].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \
     stab.h5 extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 \
diff --git a/test/Makefile.in b/test/Makefile.in
index 2ab8505..7a71fcc 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -949,10 +949,11 @@ TRACE = perl $(top_srcdir)/bin/trace
 # prefix or low-level driver with environment variables will influence
 # the temporary file name in ways that the makefile is not aware of.
 CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \
-	compact_dataset.h5 dataset.h5 dset_offset.h5 \
+	compact_dataset.h5 dataset.h5 dset_offset.h5 chunk_fixed.h5 \
 	max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \
 	huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_fast.h5 \
-	chunk_expand.h5 copy_dcpl_newfile.h5 extend.h5 istore.h5 \
+	chunk_expand.h5 layout_extend.h5 swmr_fail.h5 \
+	partial_chunks.h5 copy_dcpl_newfile.h5 extend.h5 istore.h5 \
 	extlinks*.h5 frspace.h5 links*.h5 sys_file1 tfile[1-6].h5 \
 	th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 stab.h5 extern_[1-3].h5 \
 	extern_[1-4][ab].raw gheap[0-4].h5 dt_arith[1-2] links.h5 \
diff --git a/test/dsets.c b/test/dsets.c
index 660766d..3d8ae2c 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -25,6 +25,7 @@
 
 #include "h5test.h"
 #include "H5srcdir.h"
+#include "H5Dpublic.h"
 #include "H5Vprivate.h"
 #ifdef H5_HAVE_SZLIB_H
 #   include "szlib.h"
@@ -60,7 +61,8 @@ const char *FILENAME[] = {
     "chunk_fixed",	/* 12 */
     "copy_dcpl_newfile",/* 13 */
     "partial_chunks",   /* 14 */
-    "layout_extend",
+    "layout_extend",    /* 15 */
+    "swmr_fail",        /* 16 */
     NULL
 };
 #define FILENAME_BUF_SIZE       1024
@@ -8319,9 +8321,12 @@ test_chunk_fast(hid_t fapl)
 
     /* Loop over using SWMR access to write */
     for(swmr = FALSE; swmr <= TRUE; swmr++) {
-#ifdef H5_HAVE_FILTER_DEFLATE
-        hbool_t     compress;       /* Whether chunks should be compressed */
+         hbool_t     compress;       /* Whether chunks should be compressed */
 
+        /* SWMR is only supported on the latest file format */
+        if(swmr && H5F_LIBVER_LATEST != low)
+            continue;
+#ifdef H5_HAVE_FILTER_DEFLATE
         /* Loop over compressing chunks */
         for(compress = FALSE; compress <= TRUE; compress++) {
 #endif /* H5_HAVE_FILTER_DEFLATE */
@@ -9726,7 +9731,7 @@ test_idx_compatible(void)
 
 	/* Verify index type */
 	if(idx_type != H5D_CHUNK_IDX_BTREE) 
-	    FAIL_PUTS_ERROR("should be using v1 B-tree as index");
+	    FAIL_PUTS_ERROR("should be using v1 B-tree as index")
 
 	if(H5Dclose(did) < 0) FAIL_STACK_ERROR
 
@@ -9739,7 +9744,7 @@ test_idx_compatible(void)
 
 	/* Verify index type */
 	if(idx_type != H5D_CHUNK_IDX_BTREE) 
-	    FAIL_PUTS_ERROR("should be using v1 B-tree as index");
+	    FAIL_PUTS_ERROR("should be using v1 B-tree as index")
 
 	if(H5Dclose(did) < 0) FAIL_STACK_ERROR
 
@@ -10015,6 +10020,112 @@ error:
 
 
 /*-------------------------------------------------------------------------
+ * Function: test_swmr_v1_btree_ci_fail
+ *
+ * Purpose: Tests to see if the library will disallow creation of or
+ *          writing to a version 1 B-tree under SWMR semantics.
+ *
+ * Return:      Success: 0
+ *              Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_swmr_v1_btree_ci_fail(hid_t fapl)
+{
+    char        filename[FILENAME_BUF_SIZE];
+    hid_t       fid = -1;       /* File ID */
+    hid_t       my_fapl = -1;   /* File access property list ID */
+    hid_t       dcpl = -1;      /* Dataset creation property list ID */
+    hid_t       sid = -1;       /* Dataspace ID */
+    hid_t       did = -1;       /* Dataset ID */
+    hsize_t     dims[1];        /* Size of dataset */
+    hsize_t     max_dims[1];    /* Maximum size of dataset */
+    hsize_t     chunk_dims[1];  /* Chunk dimensions */
+    herr_t      err;            /* Error return value */
+    H5D_chunk_index_t idx_type; /* Chunk index type */
+    int         data = 0;       /* Data to be written to the dataset */
+
+    TESTING("expected dataset create/write failure under SWMR using v-1 B-trees");
+
+    h5_fixname(FILENAME[16], fapl, filename, sizeof filename);
+
+    /* Copy the file access property list */
+    if((my_fapl = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR
+
+    /* Open the test file */
+    if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) FAIL_STACK_ERROR
+
+    /* Create a dataset */
+    if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR
+    chunk_dims[0] = 64;
+    if(H5Pset_chunk(dcpl, 1, chunk_dims) < 0) FAIL_STACK_ERROR
+    dims[0] = 0;
+    max_dims[0] = H5S_UNLIMITED;
+    if((sid = H5Screate_simple(1, dims, max_dims)) < 0) FAIL_STACK_ERROR
+    if((did = H5Dcreate2(fid, DSET_DEFAULT_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+
+    /* Explicitly check that the dataset is using a v-1 B-tree index */
+    if(H5D__layout_idx_type_test(did, &idx_type) < 0) FAIL_STACK_ERROR
+    if(idx_type != H5D_CHUNK_IDX_BTREE) FAIL_PUTS_ERROR("created dataset is not version 1 B-tree")
+
+    /* Close the file */
+    if(H5Dclose(did) < 0) FAIL_STACK_ERROR
+    if(H5Fclose(fid) < 0) FAIL_STACK_ERROR
+
+    /* Reopen the file with SWMR write access */
+    if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, my_fapl)) < 0) FAIL_STACK_ERROR
+
+    /* Attempt to create a dataset that uses v-1 B-tree chunk indexing under
+     * SWMR write semantics.  This will fail.
+     */
+    H5E_BEGIN_TRY {
+        did = H5Dcreate2(fid, DSET_CHUNKED_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+    } H5E_END_TRY;
+    if(did >= 0) {
+        H5_FAILED();
+        puts("    library allowed creation of a version 1 B-tree indexed dataset under SWMR semantics");
+	goto error;
+    }
+
+    /* Attempt to write to a dataset that uses v-1 B-tree chunk indexing under
+     * SWMR semantics.  Since the library can't protect a v-1 B-tree node under
+     * SWMR semantics (we have no idea if someone will try to modify it), 
+     * even opening the dataset will fail.
+     */
+    H5E_BEGIN_TRY {
+        did = H5Dopen(fid, DSET_DEFAULT_NAME, H5P_DEFAULT);
+    } H5E_END_TRY;
+    if(did >= 0) {
+        H5_FAILED();
+        puts("    library allowed opening (and potential writing) to a version 1 B-tree indexed dataset under SWMR semantics");
+        goto error;
+    }
+
+    /* Close everything */
+    if(H5Pclose(my_fapl) < 0) FAIL_STACK_ERROR
+    if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR
+    if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+    if(H5Fclose(fid) < 0) FAIL_STACK_ERROR
+
+    PASSED();
+    return 0;
+
+error:
+    return -1;
+    H5E_BEGIN_TRY {
+        H5Pclose(my_fapl);
+        H5Pclose(dcpl);
+        H5Dclose(did);
+        H5Sclose(sid);
+        H5Fclose(fid);
+    } H5E_END_TRY;
+} /* end test_swmr_v1_btree_ci_fail() */
+
+
+
+
+/*-------------------------------------------------------------------------
  * Function:    test_scatter
  *
  * Purpose:     Tests H5Dscatter with a variety of different selections
@@ -11075,6 +11186,9 @@ main(void)
     h5_reset();
     fapl = h5_fileaccess();
 
+    /* Test that SWMR access fails with version 1 B-tree access */
+    nerrors += (test_swmr_v1_btree_ci_fail(fapl) < 0 ? 1 : 0);
+
     /* Turn off the chunk cache, so all the chunks are immediately written to disk */
     if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
         goto error;
diff --git a/test/flushrefresh.c b/test/flushrefresh.c
index 223cf8d..7686fe3 100755
--- a/test/flushrefresh.c
+++ b/test/flushrefresh.c
@@ -145,6 +145,10 @@ int main(int argc, const char *argv[])
     /* Variables */
     const char *envval = NULL;
 
+    /* Initialize library */
+    if(H5open() < 0)
+        TEST_ERROR;
+
     /* Parse command line options */
     if (argc == 1) {
 
@@ -275,7 +279,7 @@ herr_t test_flush(void)
      **************************************************************************/
 
     /* Variables */
-    hid_t fid,gid,gid2,gid3,sid,tid1,tid2,tid3,did,did2,did3,rid,status = 0;
+    hid_t fid,gid,gid2,gid3,sid,tid1,tid2,tid3,did,did2,did3,rid,fapl,status = 0;
     hsize_t dims[2] = {3,5};
 
     /* Testing Message */
@@ -288,8 +292,10 @@ herr_t test_flush(void)
     /* CREATE TEST FILE */
     /* ================ */
 
-    /* Create file, open root group */
-    if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
+    /* Create file, open root group - have to use latest file format for SWMR */
+    if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR;
+    if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR; 
+    if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0) TEST_ERROR;
     if ((rid = H5Gopen2(fid, "/", H5P_DEFAULT)) < 0) TEST_ERROR;
 
     /* Create data space and types */
@@ -508,6 +514,7 @@ herr_t test_flush(void)
     /* ================== */
     /* Cleanup and Return */  
     /* ================== */
+    if (H5Pclose(fapl) < 0) TEST_ERROR;
     if (H5Gclose(gid) < 0) TEST_ERROR;
     if (H5Gclose(gid2) < 0) TEST_ERROR;
     if (H5Dclose(did) < 0) TEST_ERROR;
@@ -598,7 +605,7 @@ herr_t test_refresh(void)
      **************************************************************************/
 
     /* Variables */
-    hid_t aid,fid,sid,tid1,did,dcpl = 0;
+    hid_t aid,fid,sid,tid1,did,dcpl,fapl = 0;
     hid_t gid,gid2,gid3,tid2,tid3,did2,did3,status = 0;
     hsize_t dims[2] = {50,50};
     hsize_t cdims[2] = {1,1};
@@ -615,7 +622,9 @@ herr_t test_refresh(void)
     /* ================ */
 
     /* Create File */
-    if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
+    if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR;
+    if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR;
+    if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0) TEST_ERROR;
 
     /* Create data space and types */
     if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR;
@@ -760,6 +769,7 @@ herr_t test_refresh(void)
     /* ================== */
 
     /* Close Stuff */
+    if (H5Pclose(fapl) < 0) TEST_ERROR;
     if (H5Tclose(tid1) < 0) TEST_ERROR;
     if (H5Tclose(tid2) < 0) TEST_ERROR;
     if (H5Tclose(tid3) < 0) TEST_ERROR;
diff --git a/test/links.c b/test/links.c
index ffa4024..0bcfd31 100644
--- a/test/links.c
+++ b/test/links.c
@@ -4032,40 +4032,41 @@ external_set_elink_acc_flags(hid_t fapl, hbool_t new_format)
     if(H5Gclose(group) < 0) TEST_ERROR
     if(H5Fclose(file1) < 0) TEST_ERROR
 
-
     /* Reopen file1, with read-write and SWMR-write access */
-    if((file1 = H5Fopen(filename1, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) FAIL_STACK_ERROR
-
-    /* Open a group through the external link using default gapl */
-    if((group = H5Gopen2(file1, "/ext_link/group", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+    /* Only supported under the latest file format */
+    if(new_format) {
+        if((file1 = H5Fopen(filename1, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) FAIL_STACK_ERROR
 
-    /* Verify that the correct parameters have been set on file2 */
-    if((file2 = H5Iget_file_id(group)) < 0) FAIL_STACK_ERROR
-    if(H5Fget_intent(file2, &flags) < 0) FAIL_STACK_ERROR
-    if(flags != (H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE)) TEST_ERROR
+        /* Open a group through the external link using default gapl */
+        if((group = H5Gopen2(file1, "/ext_link/group", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
 
-    /* Close file2 and group */
-    if(H5Gclose(group) < 0) FAIL_STACK_ERROR
-    if(H5Fclose(file2) < 0) FAIL_STACK_ERROR
+        /* Verify that the correct parameters have been set on file2 */
+        if((file2 = H5Iget_file_id(group)) < 0) FAIL_STACK_ERROR
+        if(H5Fget_intent(file2, &flags) < 0) FAIL_STACK_ERROR
+        if(flags != (H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE)) TEST_ERROR
 
-    /* Set elink access flags on gapl to be H5F_ACC_RDWR (dropping SWMR_WRITE) */
-    if(H5Pset_elink_acc_flags(gapl, H5F_ACC_RDWR) < 0) FAIL_STACK_ERROR
+        /* Close file2 and group */
+        if(H5Gclose(group) < 0) FAIL_STACK_ERROR
+        if(H5Fclose(file2) < 0) FAIL_STACK_ERROR
 
-    /* Open a group through the external link using gapl */
-    if((group = H5Gopen2(file1, "/ext_link/group", gapl)) < 0) FAIL_STACK_ERROR
+        /* Set elink access flags on gapl to be H5F_ACC_RDWR (dropping SWMR_WRITE) */
+        if(H5Pset_elink_acc_flags(gapl, H5F_ACC_RDWR) < 0) FAIL_STACK_ERROR
 
-    /* Verify that the correct parameters have been set on file2 */
-    if((file2 = H5Iget_file_id(group)) < 0) FAIL_STACK_ERROR
-    if(H5Fget_intent(file2, &flags) < 0) FAIL_STACK_ERROR
-    if(flags != H5F_ACC_RDWR) TEST_ERROR
+        /* Open a group through the external link using gapl */
+        if((group = H5Gopen2(file1, "/ext_link/group", gapl)) < 0) FAIL_STACK_ERROR
 
-    /* Close file2 and group */
-    if(H5Gclose(group) < 0) FAIL_STACK_ERROR
-    if(H5Fclose(file2) < 0) FAIL_STACK_ERROR
+        /* Verify that the correct parameters have been set on file2 */
+        if((file2 = H5Iget_file_id(group)) < 0) FAIL_STACK_ERROR
+        if(H5Fget_intent(file2, &flags) < 0) FAIL_STACK_ERROR
+        if(flags != H5F_ACC_RDWR) TEST_ERROR
 
-    /* Close file1 */
-    if(H5Fclose(file1) < 0) TEST_ERROR
+        /* Close file2 and group */
+        if(H5Gclose(group) < 0) FAIL_STACK_ERROR
+        if(H5Fclose(file2) < 0) FAIL_STACK_ERROR
 
+        /* Close file1 */
+        if(H5Fclose(file1) < 0) TEST_ERROR
+    }
 
     /* Reopen file1, with read-only and SWMR-read access */
     if((file1 = H5Fopen(filename1, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0) FAIL_STACK_ERROR
diff --git a/test/swmr_generator.c b/test/swmr_generator.c
index f0fcee6..f556c3d 100644
--- a/test/swmr_generator.c
+++ b/test/swmr_generator.c
@@ -102,10 +102,14 @@ gen_skeleton(const char *filename, unsigned verbose, unsigned swmr_write,
     if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
         return -1;
 
-    /* Select the correct index type */
-    if(strcmp(index_type, "b1"))
-        if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
-            return -1;
+    /* We ALWAYS select the latest file format for SWMR */
+    if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+        return -1;
+
+    /* There are two chunk indexes tested here.
+     * With one unlimited dimension, we get the extensible array index
+     * type, with two unlimited dimensions, we get a v-2 B-tree.
+     */
     if(!strcmp(index_type, "b2"))
         max_dims[0] = H5S_UNLIMITED;
 
@@ -243,7 +247,7 @@ usage(void)
     printf("\n");
     printf("<deflate compression level> should be -1 (for no compression) or 0-9\n");
     printf("\n");
-    printf("<index type> should be b1, b2, fa, or ea (fa not yet implemented)\n");
+    printf("<index type> should be b2 or ea\n");
     printf("\n");
     printf("Defaults to verbose (no '-q' given), no SWMR_WRITE mode (no '-s' given) no\n");
     printf("compression ('-c -1'), v1 b-tree indexing (-i b1), and will generate a random\n");
@@ -280,8 +284,7 @@ int main(int argc, const char *argv[])
                     /* Chunk index type */
                     case 'i':
                         index_type = argv[u + 1];
-                        if(strcmp(index_type, "b1")
-                                && strcmp(index_type, "ea")
+                        if(strcmp(index_type, "ea")
                                 && strcmp(index_type, "b2"))
                             usage();
                         u += 2;
diff --git a/test/testswmr.sh b/test/testswmr.sh
index b178af0..794910c 100755
--- a/test/testswmr.sh
+++ b/test/testswmr.sh
@@ -69,7 +69,7 @@ while [ $# -gt 0 ]; do
 done
 
 # Loop over index types
-for index_type in "-i b1" "-i ea" "-i b2" 
+for index_type in "-i ea" "-i b2" 
 do
     # Try with and without compression
     for compress in "" "-c 5"