diff options
Diffstat (limited to 'testpar/t_chunk_alloc.c')
| -rw-r--r-- | testpar/t_chunk_alloc.c | 380 |
1 files changed, 177 insertions, 203 deletions
diff --git a/testpar/t_chunk_alloc.c b/testpar/t_chunk_alloc.c index e8a6ca4..ac5b90b 100644 --- a/testpar/t_chunk_alloc.c +++ b/testpar/t_chunk_alloc.c @@ -1,16 +1,13 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * - * the files COPYING and Copyright.html. COPYING can be found at the root * - * of the source code distribution tree; Copyright.html can be found at the * - * root level of an installed copy of the electronic HDF5 document set and * - * is linked from the top-level documents page. It can also be found at * - * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * - * access to either file, you may request a copy from help@hdfgroup.org. * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* @@ -22,27 +19,22 @@ */ #include "testphdf5.h" -static int mpi_size, mpi_rank; +static int mpi_size, mpi_rank; -#define DSET_NAME "ExtendibleArray" -#define CHUNK_SIZE 1000 /* #elements per chunk */ -#define CHUNK_FACTOR 200 /* default dataset size in terms of chunks */ -#define CLOSE 1 -#define NO_CLOSE 0 +#define DSET_NAME "ExtendibleArray" +#define CHUNK_SIZE 1000 /* #elements per chunk */ +#define CHUNK_FACTOR 200 /* default dataset size in terms of chunks */ +#define CLOSE 1 +#define NO_CLOSE 0 static MPI_Offset get_filesize(const char *filename) { - int mpierr; - MPI_File fd; - MPI_Offset filesize; -#ifndef H5_HAVE_MPI_GET_SIZE - h5_stat_t stat_buf; -#endif - -#ifdef H5_HAVE_MPI_GET_SIZE - mpierr = MPI_File_open(MPI_COMM_SELF, (char*)filename, MPI_MODE_RDONLY, - MPI_INFO_NULL, &fd); + int mpierr; + MPI_File fd; + MPI_Offset filesize; + + mpierr = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fd); VRFY((mpierr == MPI_SUCCESS), ""); mpierr = MPI_File_get_size(fd, &filesize); @@ -50,32 +42,13 @@ get_filesize(const char *filename) mpierr = MPI_File_close(&fd); VRFY((mpierr == MPI_SUCCESS), ""); -#else - /* Some systems (only SGI Altix Propack 4 so far) doesn't return correct - * file size for MPI_File_get_size. Use stat instead. - */ - if((mpierr=HDstat(filename, &stat_buf))<0) - VRFY((mpierr == MPI_SUCCESS), ""); - /* Hopefully this casting is safe */ - filesize = (MPI_Offset)(stat_buf.st_size); -#endif - - return(filesize); + return (filesize); } -typedef enum write_pattern { - none, - sec_last, - all -} write_type; - -typedef enum access_ { - write_all, - open_only, - extend_only -} access_type; +typedef enum write_pattern { none, sec_last, all } write_type; +typedef enum access_ { write_all, open_only, extend_only } access_type; /* * This creates a dataset serially with chunks, each of CHUNK_SIZE @@ -85,98 +58,97 @@ typedef enum access_ { static void create_chunked_dataset(const char *filename, int chunk_factor, write_type write_pattern) { - hid_t file_id, dataset; /* handles */ - hid_t dataspace,memspace; - hid_t cparms; - hsize_t dims[1]; - hsize_t maxdims[1] = {H5S_UNLIMITED}; - - hsize_t chunk_dims[1] ={CHUNK_SIZE}; - hsize_t count[1]; - hsize_t stride[1]; - hsize_t block[1]; - hsize_t offset[1]; /* Selection offset within dataspace */ + hid_t file_id, dataset; /* handles */ + hid_t dataspace, memspace; + hid_t cparms; + hsize_t dims[1]; + hsize_t maxdims[1] = {H5S_UNLIMITED}; + + hsize_t chunk_dims[1] = {CHUNK_SIZE}; + hsize_t count[1]; + hsize_t stride[1]; + hsize_t block[1]; + hsize_t offset[1]; /* Selection offset within dataspace */ /* Variables used in reading data back */ - char buffer[CHUNK_SIZE]; - long nchunks; - herr_t hrc; + char buffer[CHUNK_SIZE]; + long nchunks; + herr_t hrc; - MPI_Offset filesize, /* actual file size */ - est_filesize; /* estimated file size */ + MPI_Offset filesize, /* actual file size */ + est_filesize; /* estimated file size */ /* set up MPI parameters */ - MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); /* Only MAINPROCESS should create the file. Others just wait. */ - if (MAINPROCESS){ - nchunks=chunk_factor*mpi_size; - dims[0]=nchunks*CHUNK_SIZE; - /* Create the data space with unlimited dimensions. */ - dataspace = H5Screate_simple (1, dims, maxdims); - VRFY((dataspace >= 0), ""); + if (MAINPROCESS) { + nchunks = chunk_factor * mpi_size; + dims[0] = (hsize_t)(nchunks * CHUNK_SIZE); + /* Create the data space with unlimited dimensions. */ + dataspace = H5Screate_simple(1, dims, maxdims); + VRFY((dataspace >= 0), ""); - memspace = H5Screate_simple(1, chunk_dims, NULL); - VRFY((memspace >= 0), ""); + memspace = H5Screate_simple(1, chunk_dims, NULL); + VRFY((memspace >= 0), ""); - /* Create a new file. If file exists its contents will be overwritten. */ - file_id = H5Fcreate(h5_rmprefix(filename), H5F_ACC_TRUNC, H5P_DEFAULT, - H5P_DEFAULT); - VRFY((file_id >= 0), "H5Fcreate"); + /* Create a new file. If file exists its contents will be overwritten. */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + VRFY((file_id >= 0), "H5Fcreate"); - /* Modify dataset creation properties, i.e. enable chunking */ - cparms = H5Pcreate(H5P_DATASET_CREATE); - VRFY((cparms >= 0), ""); + /* Modify dataset creation properties, i.e. enable chunking */ + cparms = H5Pcreate(H5P_DATASET_CREATE); + VRFY((cparms >= 0), ""); - hrc = H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY); - VRFY((hrc >= 0), ""); + hrc = H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY); + VRFY((hrc >= 0), ""); - hrc = H5Pset_chunk(cparms, 1, chunk_dims); - VRFY((hrc >= 0), ""); + hrc = H5Pset_chunk(cparms, 1, chunk_dims); + VRFY((hrc >= 0), ""); - /* Create a new dataset within the file using cparms creation properties. */ - dataset = H5Dcreate2(file_id, DSET_NAME, H5T_NATIVE_UCHAR, dataspace, H5P_DEFAULT, cparms, H5P_DEFAULT); - VRFY((dataset >= 0), ""); + /* Create a new dataset within the file using cparms creation properties. */ + dataset = + H5Dcreate2(file_id, DSET_NAME, H5T_NATIVE_UCHAR, dataspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + VRFY((dataset >= 0), ""); - if(write_pattern == sec_last) { + if (write_pattern == sec_last) { HDmemset(buffer, 100, CHUNK_SIZE); - count[0] = 1; + count[0] = 1; stride[0] = 1; - block[0] = chunk_dims[0]; - offset[0] = (nchunks-2)*chunk_dims[0]; + block[0] = chunk_dims[0]; + offset[0] = (hsize_t)(nchunks - 2) * chunk_dims[0]; hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block); - VRFY((hrc >= 0), ""); + VRFY((hrc >= 0), ""); /* Write sec_last chunk */ hrc = H5Dwrite(dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer); VRFY((hrc >= 0), "H5Dwrite"); } /* end if */ - /* Close resources */ - hrc = H5Dclose (dataset); - VRFY((hrc >= 0), ""); - dataset = -1; - - hrc = H5Sclose (dataspace); - VRFY((hrc >= 0), ""); + /* Close resources */ + hrc = H5Dclose(dataset); + VRFY((hrc >= 0), ""); + dataset = -1; - hrc = H5Sclose (memspace); - VRFY((hrc >= 0), ""); + hrc = H5Sclose(dataspace); + VRFY((hrc >= 0), ""); - hrc = H5Pclose (cparms); - VRFY((hrc >= 0), ""); + hrc = H5Sclose(memspace); + VRFY((hrc >= 0), ""); - hrc = H5Fclose (file_id); - VRFY((hrc >= 0), ""); - file_id = -1; + hrc = H5Pclose(cparms); + VRFY((hrc >= 0), ""); - /* verify file size */ - filesize = get_filesize(filename); - est_filesize = nchunks * CHUNK_SIZE * sizeof(unsigned char); - VRFY((filesize >= est_filesize), "file size check"); + hrc = H5Fclose(file_id); + VRFY((hrc >= 0), ""); + file_id = -1; + /* verify file size */ + filesize = get_filesize(filename); + est_filesize = (MPI_Offset)nchunks * (MPI_Offset)CHUNK_SIZE * (MPI_Offset)sizeof(unsigned char); + VRFY((filesize >= est_filesize), "file size check"); } /* Make sure all processes are done before exiting this routine. Otherwise, @@ -187,7 +159,6 @@ create_chunked_dataset(const char *filename, int chunk_factor, write_type write_ MPI_Barrier(MPI_COMM_WORLD); } - /* * This program performs three different types of parallel access. It writes on * the entire dataset, it extends the dataset to nchunks*CHUNK_SIZE, and it only @@ -195,51 +166,52 @@ create_chunked_dataset(const char *filename, int chunk_factor, write_type write_ * consistent with argument 'chunk_factor'. */ static void -parallel_access_dataset(const char *filename, int chunk_factor, access_type action, hid_t *file_id, hid_t *dataset) +parallel_access_dataset(const char *filename, int chunk_factor, access_type action, hid_t *file_id, + hid_t *dataset) { /* HDF5 gubbins */ - hid_t memspace, dataspace; /* HDF5 file identifier */ - hid_t access_plist; /* HDF5 ID for file access property list */ - herr_t hrc; /* HDF5 return code */ - hsize_t size[1]; - - hsize_t chunk_dims[1] ={CHUNK_SIZE}; - hsize_t count[1]; - hsize_t stride[1]; - hsize_t block[1]; - hsize_t offset[1]; /* Selection offset within dataspace */ - hsize_t dims[1]; - hsize_t maxdims[1]; + hid_t memspace, dataspace; /* HDF5 file identifier */ + hid_t access_plist; /* HDF5 ID for file access property list */ + herr_t hrc; /* HDF5 return code */ + hsize_t size[1]; + + hsize_t chunk_dims[1] = {CHUNK_SIZE}; + hsize_t count[1]; + hsize_t stride[1]; + hsize_t block[1]; + hsize_t offset[1]; /* Selection offset within dataspace */ + hsize_t dims[1]; + hsize_t maxdims[1]; /* Variables used in reading data back */ - char buffer[CHUNK_SIZE]; - int i; - long nchunks; + char buffer[CHUNK_SIZE]; + int i; + long nchunks; /* MPI Gubbins */ - MPI_Offset filesize, /* actual file size */ - est_filesize; /* estimated file size */ + MPI_Offset filesize, /* actual file size */ + est_filesize; /* estimated file size */ /* Initialize MPI */ - MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - nchunks=chunk_factor*mpi_size; + nchunks = chunk_factor * mpi_size; /* Set up MPIO file access property lists */ - access_plist = H5Pcreate(H5P_FILE_ACCESS); + access_plist = H5Pcreate(H5P_FILE_ACCESS); VRFY((access_plist >= 0), ""); hrc = H5Pset_fapl_mpio(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL); VRFY((hrc >= 0), ""); /* Open the file */ - if (*file_id<0){ + if (*file_id < 0) { *file_id = H5Fopen(filename, H5F_ACC_RDWR, access_plist); VRFY((*file_id >= 0), ""); } /* Open dataset*/ - if (*dataset<0){ + if (*dataset < 0) { *dataset = H5Dopen2(*file_id, DSET_NAME, H5P_DEFAULT); VRFY((*dataset >= 0), ""); } @@ -250,26 +222,26 @@ parallel_access_dataset(const char *filename, int chunk_factor, access_type acti dataspace = H5Dget_space(*dataset); VRFY((dataspace >= 0), ""); - size[0] = nchunks*CHUNK_SIZE; + size[0] = (hsize_t)nchunks * CHUNK_SIZE; switch (action) { /* all chunks are written by all the processes in an interleaved way*/ case write_all: - memset(buffer, mpi_rank+1, CHUNK_SIZE); - count[0] = 1; - stride[0] = 1; - block[0] = chunk_dims[0]; - for (i=0; i<nchunks/mpi_size; i++){ - offset[0] = (i*mpi_size+mpi_rank)*chunk_dims[0]; + HDmemset(buffer, mpi_rank + 1, CHUNK_SIZE); + count[0] = 1; + stride[0] = 1; + block[0] = chunk_dims[0]; + for (i = 0; i < nchunks / mpi_size; i++) { + offset[0] = (hsize_t)(i * mpi_size + mpi_rank) * chunk_dims[0]; - hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block); - VRFY((hrc >= 0), ""); + hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block); + VRFY((hrc >= 0), ""); - /* Write the buffer out */ - hrc = H5Dwrite(*dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer); - VRFY((hrc >= 0), "H5Dwrite"); + /* Write the buffer out */ + hrc = H5Dwrite(*dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer); + VRFY((hrc >= 0), "H5Dwrite"); } break; @@ -289,8 +261,9 @@ parallel_access_dataset(const char *filename, int chunk_factor, access_type acti /* only opens the *dataset */ case open_only: - break; + default: + HDassert(0); } /* Close up */ @@ -298,10 +271,10 @@ parallel_access_dataset(const char *filename, int chunk_factor, access_type acti VRFY((hrc >= 0), ""); *dataset = -1; - hrc = H5Sclose (dataspace); + hrc = H5Sclose(dataspace); VRFY((hrc >= 0), ""); - hrc = H5Sclose (memspace); + hrc = H5Sclose(memspace); VRFY((hrc >= 0), ""); hrc = H5Fclose(*file_id); @@ -309,8 +282,8 @@ parallel_access_dataset(const char *filename, int chunk_factor, access_type acti *file_id = -1; /* verify file size */ - filesize = get_filesize(filename); - est_filesize = nchunks*CHUNK_SIZE*sizeof(unsigned char); + filesize = get_filesize(filename); + est_filesize = (MPI_Offset)nchunks * (MPI_Offset)CHUNK_SIZE * (MPI_Offset)sizeof(unsigned char); VRFY((filesize >= est_filesize), "file size check"); /* Can close some plists */ @@ -333,44 +306,45 @@ parallel_access_dataset(const char *filename, int chunk_factor, access_type acti * interleaved pattern. */ static void -verify_data(const char *filename, int chunk_factor, write_type write_pattern, int close, hid_t *file_id, hid_t *dataset) +verify_data(const char *filename, int chunk_factor, write_type write_pattern, int vclose, hid_t *file_id, + hid_t *dataset) { /* HDF5 gubbins */ - hid_t dataspace, memspace; /* HDF5 file identifier */ - hid_t access_plist; /* HDF5 ID for file access property list */ - herr_t hrc; /* HDF5 return code */ - - hsize_t chunk_dims[1] ={CHUNK_SIZE}; - hsize_t count[1]; - hsize_t stride[1]; - hsize_t block[1]; - hsize_t offset[1]; /* Selection offset within dataspace */ + hid_t dataspace, memspace; /* HDF5 file identifier */ + hid_t access_plist; /* HDF5 ID for file access property list */ + herr_t hrc; /* HDF5 return code */ + + hsize_t chunk_dims[1] = {CHUNK_SIZE}; + hsize_t count[1]; + hsize_t stride[1]; + hsize_t block[1]; + hsize_t offset[1]; /* Selection offset within dataspace */ /* Variables used in reading data back */ - char buffer[CHUNK_SIZE]; - int value, i; - int index; - long nchunks; + char buffer[CHUNK_SIZE]; + int value, i; + int index_l; + long nchunks; /* Initialize MPI */ - MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - nchunks=chunk_factor*mpi_size; + nchunks = chunk_factor * mpi_size; /* Set up MPIO file access property lists */ - access_plist = H5Pcreate(H5P_FILE_ACCESS); + access_plist = H5Pcreate(H5P_FILE_ACCESS); VRFY((access_plist >= 0), ""); hrc = H5Pset_fapl_mpio(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL); VRFY((hrc >= 0), ""); /* Open the file */ - if (*file_id<0){ + if (*file_id < 0) { *file_id = H5Fopen(filename, H5F_ACC_RDWR, access_plist); VRFY((*file_id >= 0), ""); } /* Open dataset*/ - if (*dataset<0){ + if (*dataset < 0) { *dataset = H5Dopen2(*file_id, DSET_NAME, H5P_DEFAULT); VRFY((*dataset >= 0), ""); } @@ -382,14 +356,14 @@ verify_data(const char *filename, int chunk_factor, write_type write_pattern, in VRFY((dataspace >= 0), ""); /* all processes check all chunks. */ - count[0] = 1; + count[0] = 1; stride[0] = 1; - block[0] = chunk_dims[0]; - for (i=0; i<nchunks; i++){ - /* reset buffer values */ - memset(buffer, -1, CHUNK_SIZE); + block[0] = chunk_dims[0]; + for (i = 0; i < nchunks; i++) { + /* reset buffer values */ + HDmemset(buffer, -1, CHUNK_SIZE); - offset[0] = i*chunk_dims[0]; + offset[0] = (hsize_t)i * chunk_dims[0]; hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block); VRFY((hrc >= 0), ""); @@ -399,37 +373,40 @@ verify_data(const char *filename, int chunk_factor, write_type write_pattern, in VRFY((hrc >= 0), "H5Dread"); /* set expected value according the write pattern */ - switch (write_pattern) { - case all: - value = i%mpi_size + 1; - break; - case none: - value = 0; - break; + switch (write_pattern) { + case all: + value = i % mpi_size + 1; + break; + case none: + value = 0; + break; case sec_last: - if (i==nchunks-2) - value = 100; - else - value = 0; - } + if (i == nchunks - 2) + value = 100; + else + value = 0; + break; + default: + HDassert(0); + } /* verify content of the chunk */ - for (index = 0; index < CHUNK_SIZE; index++) - VRFY((buffer[index] == value), "data verification"); + for (index_l = 0; index_l < CHUNK_SIZE; index_l++) + VRFY((buffer[index_l] == value), "data verification"); } - hrc = H5Sclose (dataspace); - VRFY((hrc >= 0), ""); + hrc = H5Sclose(dataspace); + VRFY((hrc >= 0), ""); - hrc = H5Sclose (memspace); - VRFY((hrc >= 0), ""); + hrc = H5Sclose(memspace); + VRFY((hrc >= 0), ""); /* Can close some plists */ hrc = H5Pclose(access_plist); VRFY((hrc >= 0), ""); /* Close up */ - if (close){ + if (vclose) { hrc = H5Dclose(*dataset); VRFY((hrc >= 0), ""); *dataset = -1; @@ -446,8 +423,6 @@ verify_data(const char *filename, int chunk_factor, write_type write_pattern, in MPI_Barrier(MPI_COMM_WORLD); } - - /* * Test following possible scenarios, * Case 1: @@ -470,17 +445,17 @@ void test_chunk_alloc(void) { const char *filename; - hid_t file_id, dataset; + hid_t file_id, dataset; file_id = dataset = -1; /* Initialize MPI */ - MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - filename = GetTestParameters(); + filename = (const char *)GetTestParameters(); if (VERBOSE_MED) - printf("Extend Chunked allocation test on file %s\n", filename); + HDprintf("Extend Chunked allocation test on file %s\n", filename); /* Case 1 */ /* Create chunked dataset without writing anything.*/ @@ -507,5 +482,4 @@ test_chunk_alloc(void) parallel_access_dataset(filename, CHUNK_FACTOR, write_all, &file_id, &dataset); /* reopen dataset in parallel, read and verify the data */ verify_data(filename, CHUNK_FACTOR, all, CLOSE, &file_id, &dataset); - } |