summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlbert Cheng <acheng@hdfgroup.org>2005-08-30 04:35:17 (GMT)
committerAlbert Cheng <acheng@hdfgroup.org>2005-08-30 04:35:17 (GMT)
commit3182a8a040b463629fd9f266b6a1eda6ce70c3d7 (patch)
treedb107c21fe8ae27e83b05686fd919dd44c0c12b6
parentbd4312d0497592a665a25415ec2f9d3db9d9f6b2 (diff)
downloadhdf5-3182a8a040b463629fd9f266b6a1eda6ce70c3d7.zip
hdf5-3182a8a040b463629fd9f266b6a1eda6ce70c3d7.tar.gz
hdf5-3182a8a040b463629fd9f266b6a1eda6ce70c3d7.tar.bz2
[svn-r11312] Purpose:
Bug #281 Description: A dataset created in serial mode with H5D_ALLOC_TIME_INCR allocation setting was not extendible, either explicitly by H5Dextend or implicitly by writing to unallocated chunks. This was because parallel mode expects the allocation mode be H5D_ALLOC_TIME_INCR only. Solution: Modified library to allocate more space when needed or directed if the file is opened by parallel mode, independent of what the dataset allocation mode is. Platforms tested: Heping pp.
-rw-r--r--testpar/Makefile.am2
-rw-r--r--testpar/Makefile.in6
-rw-r--r--testpar/t_chunk_alloc.c194
-rw-r--r--testpar/testphdf5.c2
-rw-r--r--testpar/testphdf5.h1
5 files changed, 202 insertions, 3 deletions
diff --git a/testpar/Makefile.am b/testpar/Makefile.am
index 92985b8..7c4f567 100644
--- a/testpar/Makefile.am
+++ b/testpar/Makefile.am
@@ -31,7 +31,7 @@ check_PROGRAMS = $(TEST_PROG_PARA)
check_SCRIPTS= $(TEST_SCRIPT)
testphdf5_SOURCES=testphdf5.c t_dset.c t_file.c t_mdset.c t_ph5basic.c \
- t_coll_chunk.c t_span_tree.c
+ t_coll_chunk.c t_span_tree.c t_chunk_alloc.c
# The tests all depend on the hdf5 library and the test library
LDADD = $(LIBHDF5) $(LIBH5TEST)
diff --git a/testpar/Makefile.in b/testpar/Makefile.in
index 0a7aea4..c886e66 100644
--- a/testpar/Makefile.in
+++ b/testpar/Makefile.in
@@ -78,7 +78,8 @@ t_mpi_LDADD = $(LDADD)
t_mpi_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
am_testphdf5_OBJECTS = testphdf5.$(OBJEXT) t_dset.$(OBJEXT) \
t_file.$(OBJEXT) t_mdset.$(OBJEXT) t_ph5basic.$(OBJEXT) \
- t_coll_chunk.$(OBJEXT) t_span_tree.$(OBJEXT)
+ t_coll_chunk.$(OBJEXT) t_span_tree.$(OBJEXT) \
+ t_chunk_alloc.$(OBJEXT)
testphdf5_OBJECTS = $(am_testphdf5_OBJECTS)
testphdf5_LDADD = $(LDADD)
testphdf5_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
@@ -321,7 +322,7 @@ TEST_PROG_PARA = t_mpi t_fphdf5 testphdf5
TEST_SCRIPT_PARA = testph5.sh
check_SCRIPTS = $(TEST_SCRIPT)
testphdf5_SOURCES = testphdf5.c t_dset.c t_file.c t_mdset.c t_ph5basic.c \
- t_coll_chunk.c t_span_tree.c
+ t_coll_chunk.c t_span_tree.c t_chunk_alloc.c
# The tests all depend on the hdf5 library and the test library
@@ -403,6 +404,7 @@ mostlyclean-compile:
distclean-compile:
-rm -f *.tab.c
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/t_chunk_alloc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/t_coll_chunk.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/t_dset.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/t_file.Po@am__quote@
diff --git a/testpar/t_chunk_alloc.c b/testpar/t_chunk_alloc.c
new file mode 100644
index 0000000..cd56c40
--- /dev/null
+++ b/testpar/t_chunk_alloc.c
@@ -0,0 +1,194 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * This verifies if the storage space allocation methods are compatible between
+ * serial and parallel modes.
+ *
+ * Created by: Christian Chilan and Albert Cheng
+ * Date: 2005/07/05
+ */
+
+#include "testphdf5.h"
+
+#define DATASETNAME "ExtendibleArray"
+#define ELEMS 20000000
+
+/*
+ * This creates a dataset serially with 20000 chunks, each of 1000
+ * elements. It does not perform any writing on it. Another routine
+ * will open this in parallel for extension test.
+ */
+void
+create_chunked_dataset(const char *filename)
+{
+ hid_t file; /* handles */
+ hid_t dataspace, dataset;
+ hid_t cparms;
+ int mpi_size, mpi_rank;
+ hsize_t dims[1] = {20*1000000}; /* dataset dimensions
+ at creation time */
+
+ hsize_t maxdims[1] = {H5S_UNLIMITED};
+ herr_t hrc;
+
+ /* Variables used in reading data back */
+ hsize_t chunk_dims[1] ={1000};
+
+ /* set up MPI parameters */
+ MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
+ MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
+
+ /* Create the data space with unlimited dimensions. */
+ dataspace = H5Screate_simple (1, dims, maxdims);
+ VRFY((dataspace >= 0), "");
+
+ /* Create a new file. If file exists its contents will be overwritten. */
+ file = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+ VRFY((file >= 0), "");
+
+ /* Modify dataset creation properties, i.e. enable chunking */
+ cparms = H5Pcreate (H5P_DATASET_CREATE);
+ VRFY((cparms >= 0), "");
+
+ hrc = H5Pset_chunk ( cparms, 1, chunk_dims);
+ VRFY((hrc >= 0), "");
+
+ /* Create a new dataset within the file using cparms creation properties. */
+ dataset = H5Dcreate (file, DATASETNAME, H5T_NATIVE_INT, dataspace, cparms);
+ VRFY((dataset >= 0), "");
+
+ /* Close resources */
+ hrc = H5Dclose (dataset);
+ VRFY((hrc >= 0), "");
+
+ hrc = H5Fclose (file);
+ VRFY((hrc >= 0), "");
+}
+
+/*
+ * This program extends the size of a small dataset to 20000 chunks (each of
+ * 1000 elements) in parallel. The dataset was initially created in a serial
+ * environment. A correct extend operation should increase the file size
+ * considerably.
+ */
+void
+extend_chunked_dataset(const char *filename)
+{
+ /* HDF5 gubbins */
+ hid_t file_id, dataset; /* HDF5 file identifier */
+ hid_t access_plist; /* HDF5 ID for file access property list */
+ herr_t hrc; /* HDF5 return code */
+
+ hsize_t size[1];
+ int mpi_size, mpi_rank;
+
+ long int lb;
+
+ /* MPI Gubbins */
+ MPI_Comm comm;
+ MPI_Info info;
+ MPI_File thefile;
+ MPI_Offset filesize;
+
+ int mpierr, ret;
+
+
+printf("extend_chunked_dataset: filename=%s\n", filename);
+ /* Initialize MPI */
+ comm = MPI_COMM_WORLD;
+ info = MPI_INFO_NULL;
+
+ /* set up MPI parameters */
+ MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
+ MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
+
+ /* Set up HDF5 file creation, access and independent data transfer property lists */
+ access_plist = H5Pcreate(H5P_FILE_ACCESS);
+ VRFY((access_plist >= 0), "");
+
+ hrc = H5Pset_fapl_mpio(access_plist, comm, info);
+ VRFY((hrc >= 0), "");
+
+ /* Open the file */
+ file_id = H5Fopen(filename, H5F_ACC_RDWR, access_plist);
+ VRFY((file_id >= 0), "");
+
+ /* Can close some plists */
+ hrc = H5Pclose(access_plist);
+ VRFY((hrc >= 0), "");
+
+ /* Open dataset*/
+ dataset = H5Dopen(file_id, DATASETNAME);
+ VRFY((dataset >= 0), "");
+
+ size[0] = ELEMS;
+
+ /* Extend dataset*/
+ hrc = H5Dextend(dataset, size);
+ VRFY((hrc >= 0), "");
+
+ /* Close up */
+ hrc = H5Dclose(dataset);
+ VRFY((hrc >= 0), "");
+
+ hrc = H5Fclose(file_id);
+ VRFY((hrc >= 0), "");
+
+ mpierr = MPI_File_open(comm, filename, MPI_MODE_RDONLY, info, &thefile);
+ VRFY((mpierr == MPI_SUCCESS), "");
+
+ mpierr = MPI_File_get_size(thefile, &filesize);
+ VRFY((mpierr == MPI_SUCCESS), "");
+
+
+ /* Definition of lower bound */
+ lb = ELEMS*sizeof(H5T_NATIVE_INT);
+
+ /* Print size of file. */
+ printf("File size: %ld\n", (long int)filesize);
+
+ if (filesize>=lb){
+ printf("Test passed\n");
+ ret = 0;
+ } else {
+ printf("Test failed\n");
+ ret = 1;
+ }
+
+ mpierr = MPI_File_close(&thefile);
+ VRFY((mpierr == MPI_SUCCESS), "");
+
+ hrc = H5close();
+ VRFY((hrc >= 0), "");
+
+ /* Finalize */
+ mpierr = MPI_Finalize();
+ VRFY((mpierr == MPI_SUCCESS), "");
+}
+
+void
+test_chunk_alloc(void)
+{
+ const char *filename;
+
+ filename = GetTestParameters();
+ if (VERBOSE_MED)
+ printf("Extend Chunked allocation test on file %s\n", filename);
+
+ /* create the datafile first */
+ create_chunked_dataset(filename);
+ /* reopen it in parallel and extend it. */
+ /*extend_chunked_dataset(filename); */
+}
diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c
index 2f75e51..10ce989 100644
--- a/testpar/testphdf5.c
+++ b/testpar/testphdf5.c
@@ -381,6 +381,8 @@ int main(int argc, char **argv)
"extendible dataset collective read", PARATESTFILE);
AddTest("eidsetw2", extend_writeInd2, NULL,
"extendible dataset independent write #2", PARATESTFILE);
+ AddTest("calloc", test_chunk_alloc, NULL,
+ "parallel extend Chunked allocation on serial file", PARATESTFILE);
#ifdef H5_HAVE_FILTER_DEFLATE
AddTest("cmpdsetr", compress_readAll, NULL,
diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h
index 14e315e..e085b62 100644
--- a/testpar/testphdf5.h
+++ b/testpar/testphdf5.h
@@ -187,6 +187,7 @@ void dataset_readInd(void);
void dataset_readAll(void);
void extend_readInd(void);
void extend_readAll(void);
+void test_chunk_alloc(void);
void compact_dataset(void);
void null_dataset(void);
void big_dataset(void);