[svn-r13788] Test parallel read of dataset written serially using several (combinations of) filters. Tested on kagiso and copper.

1 files changed, 465 insertions, 0 deletions

diff --git a/testpar/t_filter_read.c b/testpar/t_filter_read.c
new file mode 100644
index 0000000..1e76c46
--- /dev/null
+++ b/testpar/t_filter_read.c
@@ -0,0 +1,465 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the files COPYING and Copyright.html.  COPYING can be found at the root   *
+ * of the source code distribution tree; Copyright.html can be found at the  *
+ * root level of an installed copy of the electronic HDF5 document set and   *
+ * is linked from the top-level documents page.  It can also be found at     *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have          *
+ * access to either file, you may request a copy from help@hdfgroup.org.     *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * This verifies the correctness of parallel reading of a dataset that has been
+ * written serially using filters. 
+ *
+ * Created by: Christian Chilan
+ * Date: 2007/05/15
+ */
+
+
+#include "testphdf5.h"
+
+#ifdef H5_HAVE_SZLIB_H
+#   include "szlib.h"
+#endif
+
+static int mpi_size, mpi_rank;
+
+/* Chunk sizes */
+#define CHUNK_DIM1       7 
+#define CHUNK_DIM2       27
+
+/* Sizes of the vertical hyperslabs. Total dataset size is
+   {HS_DIM1, HS_DIM2 * mpi_size } */  
+#define HS_DIM1       200
+#define HS_DIM2       100
+
+
+/*-------------------------------------------------------------------------
+ * Function:	filter_read_internal
+ *
+ * Purpose:     Tests parallel reading of a 2D dataset written serially using
+ *              filters. During the parallel reading phase, the dataset is 
+ *              divided evenly among the processors in vertical hyperslabs. 
+ *
+ * Programmer:  Christian Chilan 
+ *              Tuesday, May 15, 2007 
+ *
+ *-------------------------------------------------------------------------
+ */
+static void 
+filter_read_internal(const char *filename, hid_t dcpl, 
+                     hsize_t *dset_size)
+{
+    hid_t		file, dataset;        /* HDF5 IDs */
+    hid_t		access_plist;         /* Access property list ID */
+    hid_t		sid, memspace;            /* Dataspace IDs */
+    hsize_t	        size[2];           /* Dataspace dimensions */
+    hsize_t	        hs_offset[2]; /* Hyperslab offset */
+    hsize_t	        hs_size[2];   /* Hyperslab size */
+    size_t		i, j;        /* Local index variables */
+    char                name[32] = "dataset";
+    herr_t              hrc;         /* Error status */
+    int                 *points = NULL; /* Writing buffer for entire dataset */ 
+    int                 *check = NULL; /* Reading buffer for selected hyperslab */
+
+    /* set up MPI parameters */
+    MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
+    MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
+
+    /* set sizes for dataset and hyperslabs */ 
+    hs_size[0] = size[0] = HS_DIM1;
+    hs_size[1] = HS_DIM2;
+
+    size[1] = hs_size[1] * mpi_size;
+
+    hs_offset[0] = 0;
+    hs_offset[1] = hs_size[1] * mpi_rank;
+    
+    /* Create the data space */
+    sid = H5Screate_simple(2, size, NULL);
+    VRFY(sid>=0, "H5Screate_simple");
+
+    /* Create buffers */
+    points = (int *)HDmalloc(size[0]*size[1]*sizeof(int));
+    VRFY(points!=NULL, "HDmalloc");
+
+    check = (int *)HDmalloc(hs_size[0]*hs_size[1]*sizeof(int)); 
+    VRFY(check!=NULL, "HDmalloc");
+
+    /* Initialize writing buffer with random data */
+    for (i=0; i<size[0]; i++) {
+        for (j=0; j<size[1]; j++) {
+            points[i*size[1]+j] = (int)(i+j+7);
+        }
+    }
+
+    VRFY(H5Pall_filters_avail(dcpl), "Incorrect filter availability");
+
+    /* Serial write phase */
+    if (MAINPROCESS){
+
+        file = H5Fcreate(h5_rmprefix(filename), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+        VRFY(file>=0, "H5Fcreate");
+
+        /* Create the dataset */
+        dataset = H5Dcreate(file, name, H5T_NATIVE_INT, sid, dcpl);
+        VRFY(dataset>=0, "H5Dcreate");
+
+        hrc = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points);
+        VRFY(hrc>=0, "H5Dwrite");
+
+        *dset_size=H5Dget_storage_size(dataset);
+        VRFY(*dset_size>0, "H5Dget_storage_size");
+
+        hrc = H5Dclose (dataset);
+        VRFY(hrc>=0, "H5Dclose");
+
+        hrc = H5Fclose (file);
+        VRFY(hrc>=0, "H5Fclose");
+    }
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Parallel read phase */ 
+    /* Set up MPIO file access property lists */
+    access_plist  = H5Pcreate(H5P_FILE_ACCESS);
+    VRFY((access_plist >= 0), "H5Pcreate");
+
+    hrc = H5Pset_fapl_mpio(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL);
+    VRFY((hrc >= 0), "H5Pset_fapl_mpio");
+
+    /* Open the file */
+    file = H5Fopen(filename, H5F_ACC_RDWR, access_plist);
+    VRFY((file >= 0), "H5Fopen");
+
+    dataset = H5Dopen (file, name);
+    VRFY((dataset >= 0), "H5Dopen");
+
+    hrc = H5Sselect_hyperslab(sid, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL);
+    VRFY(hrc>=0, "H5Sselect_hyperslab");
+
+    memspace = H5Screate_simple(2, hs_size, NULL);
+    VRFY(memspace>=0, "H5Screate_simple");
+ 
+    hrc = H5Dread (dataset, H5T_NATIVE_INT, memspace, sid, H5P_DEFAULT, check);
+    VRFY(hrc>=0, "H5Dread");
+
+    /* Check that the values read are the same as the values written */
+    for (i=0; i<hs_size[0]; i++) {
+        for (j=0; j<hs_size[1]; j++) {
+            if (points[i*size[1]+(size_t)hs_offset[1]+j] !=
+                      check[i*hs_size[1]+j]) {
+		  fprintf(stderr,"    Read different values than written.\n");
+		  fprintf(stderr,"    At index %lu,%lu\n",
+		         (unsigned long)(i),
+		         (unsigned long)(hs_offset[1]+j));
+		  fprintf(stderr,"    At original: %d\n",
+		         (int)points[i*size[1]+(size_t)hs_offset[1]+j]);
+		  fprintf(stderr,"    At returned: %d\n",
+		         (int)check[i*hs_size[1]+j]);
+                  VRFY(FALSE, "");
+	    }
+	}
+    }
+
+    /* Get the storage size of the dataset */
+    *dset_size=H5Dget_storage_size(dataset);
+    VRFY(*dset_size!=0, "H5Dget_storage_size");
+
+    /* Clean up objects used for this test */
+    hrc = H5Dclose (dataset);
+    VRFY(hrc>=0, "H5Dclose");
+
+    hrc = H5Sclose (sid);
+    VRFY(hrc>=0, "H5Sclose");
+
+    hrc = H5Sclose (memspace);
+    VRFY(hrc>=0, "H5Sclose");
+    
+    hrc = H5Pclose (access_plist);
+    VRFY(hrc>=0, "H5Pclose");
+    
+    hrc = H5Fclose (file);
+    VRFY(hrc>=0, "H5Fclose");
+   
+    free(points);
+    free(check);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function:    test_filter_read	
+ *
+ * Purpose:	Tests parallel reading of datasets written serially using
+ *              several (combinations of) filters. 
+ *
+ * Programmer:	Christian Chilan
+ *              Tuesday, May 15, 2007
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+void
+test_filter_read(void)
+{
+    hid_t	dc,file;                 /* HDF5 IDs */
+    const hsize_t chunk_size[2] = {CHUNK_DIM1, CHUNK_DIM2};  /* Chunk dimensions */
+    hsize_t     null_size;          /* Size of dataset without filters */
+    herr_t      hrc;
+    const char *filename;
+#ifdef H5_HAVE_FILTER_FLETCHER32
+    hsize_t     fletcher32_size;       /* Size of dataset with Fletcher32 checksum */
+#endif /* H5_HAVE_FILTER_FLETCHER32 */
+
+#ifdef H5_HAVE_FILTER_DEFLATE
+    hsize_t     deflate_size;       /* Size of dataset with deflate filter */
+#endif /* H5_HAVE_FILTER_DEFLATE */
+
+#ifdef H5_HAVE_FILTER_SZIP
+    hsize_t     szip_size;       /* Size of dataset with szip filter */
+    unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK;
+    unsigned szip_pixels_per_block=4;
+#endif /* H5_HAVE_FILTER_SZIP */
+
+#ifdef H5_HAVE_FILTER_SHUFFLE
+    hsize_t     shuffle_size;       /* Size of dataset with shuffle filter */
+#endif /* H5_HAVE_FILTER_SHUFFLE */
+
+#if (defined H5_HAVE_FILTER_DEFLATE | defined H5_HAVE_FILTER_SZIP) && defined H5_HAVE_FILTER_SHUFFLE && defined H5_HAVE_FILTER_FLETCHER32
+    hsize_t     combo_size;     /* Size of dataset with shuffle+deflate filter */
+#endif /* H5_HAVE_FILTER_DEFLATE && H5_HAVE_FILTER_SHUFFLE && H5_HAVE_FILTER_FLETCHER32 */
+
+    filename = GetTestParameters();
+
+    if (VERBOSE_MED)
+        printf("Parallel reading of dataset written with filters %s\n", filename);
+
+    /*----------------------------------------------------------
+     * STEP 0: Test without filters.
+     *----------------------------------------------------------
+     */
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0,"H5Pcreate"); 
+
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0,"H5Pset_chunk"); 
+
+    filter_read_internal(filename,dc,&null_size);
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0,"H5Pclose"); 
+
+    /*----------------------------------------------------------
+     * STEP 1: Test Fletcher32 Checksum by itself.
+     *----------------------------------------------------------
+     */
+#ifdef H5_HAVE_FILTER_FLETCHER32
+
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0,"H5Pset_filter"); 
+   
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0,"H5Pset_filter"); 
+   
+    hrc = H5Pset_filter (dc,H5Z_FILTER_FLETCHER32,0,0,NULL);
+    VRFY(hrc>=0,"H5Pset_filter"); 
+
+    filter_read_internal(filename,dc,&fletcher32_size);
+    VRFY(fletcher32_size > null_size,"Size after checksumming is incorrect.");
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+#endif /* H5_HAVE_FILTER_FLETCHER32 */
+
+    /*----------------------------------------------------------
+     * STEP 2: Test deflation by itself.
+     *----------------------------------------------------------
+     */
+#ifdef H5_HAVE_FILTER_DEFLATE
+
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0, "H5Pcreate");
+
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0, "H5Pset_chunk");
+
+    hrc = H5Pset_deflate (dc, 6);
+    VRFY(hrc>=0, "H5Pset_deflate");
+ 
+    filter_read_internal(filename,dc,&deflate_size);
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+#endif /* H5_HAVE_FILTER_DEFLATE */
+
+    /*----------------------------------------------------------
+     * STEP 3: Test szip compression by itself.
+     *----------------------------------------------------------
+     */
+#ifdef H5_HAVE_FILTER_SZIP
+    if ( h5_szip_can_encode() == 1) {
+        dc = H5Pcreate(H5P_DATASET_CREATE);
+        VRFY(dc>=0, "H5Pcreate");
+
+        hrc = H5Pset_chunk (dc, 2, chunk_size);
+        VRFY(hrc>=0, "H5Pset_chunk");
+
+	hrc = H5Pset_szip(dc, szip_options_mask, szip_pixels_per_block);
+        VRFY(hrc>=0, "H5Pset_szip");
+
+	filter_read_internal(filename,dc,&szip_size);
+
+        /* Clean up objects used for this test */
+        hrc = H5Pclose (dc);
+        VRFY(hrc>=0, "H5Pclose");
+    }
+#endif /* H5_HAVE_FILTER_SZIP */
+
+    /*----------------------------------------------------------
+     * STEP 4: Test shuffling by itself.
+     *----------------------------------------------------------
+     */
+#ifdef H5_HAVE_FILTER_SHUFFLE
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0, "H5Pcreate");
+
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0, "H5Pset_chunk");
+
+    hrc = H5Pset_shuffle (dc);
+    VRFY(hrc>=0, "H5Pset_shuffle");
+
+    filter_read_internal(filename,dc,&shuffle_size);
+    VRFY(shuffle_size==null_size,"Shuffled size not the same as uncompressed size.");
+    
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+#endif /* H5_HAVE_FILTER_SHUFFLE */
+
+    /*----------------------------------------------------------
+     * STEP 5: Test shuffle + deflate + checksum in any order.
+     *----------------------------------------------------------
+     */
+#if defined H5_HAVE_FILTER_DEFLATE && defined H5_HAVE_FILTER_SHUFFLE && defined H5_HAVE_FILTER_FLETCHER32
+    /* Testing shuffle+deflate+checksum filters (checksum first) */
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0, "H5Pcreate");
+
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0, "H5Pset_chunk");
+ 
+    hrc = H5Pset_fletcher32 (dc);
+    VRFY(hrc>=0, "H5Pset_fletcher32");
+ 
+    hrc = H5Pset_shuffle (dc);
+    VRFY(hrc>=0, "H5Pset_shuffle");
+    
+    hrc = H5Pset_deflate (dc, 6);
+    VRFY(hrc>=0, "H5Pset_deflate");
+
+    filter_read_internal(filename,dc,&combo_size);
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+    /* Testing shuffle+deflate+checksum filters (checksum last) */
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0, "H5Pcreate");
+    
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0, "H5Pset_chunk");
+
+    hrc = H5Pset_shuffle (dc);
+    VRFY(hrc>=0, "H5Pset_shuffle");
+
+    hrc = H5Pset_deflate (dc, 6);
+    VRFY(hrc>=0, "H5Pset_deflate");
+
+    hrc = H5Pset_fletcher32 (dc);
+    VRFY(hrc>=0, "H5Pset_fletcher32");
+
+    filter_read_internal(filename,dc,&combo_size);
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+#endif /* H5_HAVE_FILTER_DEFLATE && H5_HAVE_FILTER_SHUFFLE && H5_HAVE_FILTER_FLETCHER32 */
+
+    /*----------------------------------------------------------
+     * STEP 6: Test shuffle + szip + checksum in any order.
+     *----------------------------------------------------------
+     */
+#if defined H5_HAVE_FILTER_SZIP && defined H5_HAVE_FILTER_SHUFFLE && defined H5_HAVE_FILTER_FLETCHER32
+
+    /* Testing shuffle+szip(with encoder)+checksum filters(checksum first) */ 
+    dc = H5Pcreate(H5P_DATASET_CREATE);
+    VRFY(dc>=0, "H5Pcreate");
+
+    hrc = H5Pset_chunk (dc, 2, chunk_size);
+    VRFY(hrc>=0, "H5Pset_chunk");
+
+    hrc = H5Pset_fletcher32 (dc);
+    VRFY(hrc>=0, "H5Pset_fletcher32");
+
+    hrc = H5Pset_shuffle (dc);
+    VRFY(hrc>=0, "H5Pset_shuffle");
+
+    /* Make sure encoding is enabled */
+    if ( h5_szip_can_encode() == 1) {
+	hrc = H5Pset_szip(dc, szip_options_mask, szip_pixels_per_block);
+        VRFY(hrc>=0, "H5Pset_szip");
+
+	filter_read_internal(filename,dc,&combo_size);
+    }
+
+    /* Clean up objects used for this test */
+    hrc = H5Pclose (dc);
+    VRFY(hrc>=0, "H5Pclose");
+
+    /* Testing shuffle+szip(with encoder)+checksum filters(checksum last) */ 
+    /* Make sure encoding is enabled */
+    if ( h5_szip_can_encode() == 1) {
+	dc = H5Pcreate(H5P_DATASET_CREATE);
+        VRFY(dc>=0, "H5Pcreate");
+
+	hrc = H5Pset_chunk (dc, 2, chunk_size);
+        VRFY(hrc>=0, "H5Pset_chunk");
+
+	hrc = H5Pset_shuffle (dc);
+        VRFY(hrc>=0, "H5Pset_shuffle");
+
+	hrc = H5Pset_szip(dc, szip_options_mask, szip_pixels_per_block);
+        VRFY(hrc>=0, "H5Pset_szip");
+
+	hrc = H5Pset_fletcher32 (dc);
+        VRFY(hrc>=0, "H5Pset_fletcher32");
+
+	filter_read_internal(filename,dc,&combo_size);
+
+	/* Clean up objects used for this test */
+	hrc = H5Pclose (dc);
+        VRFY(hrc>=0, "H5Pclose");
+    }
+
+#endif /* H5_HAVE_FILTER_SZIP && H5_HAVE_FILTER_SHUFFLE && H5_HAVE_FILTER_FLETCHER32 */
+}
+