From 5d6ed0b298e3a40051a975a466820030360e8a31 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 3 Jul 2017 14:26:53 -0700 Subject: Turned off unused parameter warnings in gcc for Java constants. None of the constant functions use their parameters, so emitting warnings for them generates a large amount of noise. Since these are trivial wrappers, marking them up with H5_ATTR_UNUSED would be a huge waste of time. --- java/src/jni/h5Constants.c | 1 + 1 file changed, 1 insertion(+) diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index d4511e1..f6f8bfa 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -26,6 +26,7 @@ extern "C" { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-prototypes" +#pragma GCC diagnostic ignored "-Wunused-parameter" JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5_1QUARTER_1HADDR_1MAX(JNIEnv *env, jclass cls) { return (hsize_t)HADDR_MAX/4; } -- cgit v0.12 From 191147ec90d72619632a24242c003ecb3bccfffd Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 5 Jul 2017 09:35:24 -0700 Subject: * Yanked the smoke check that was recently introduced to test the unused H5I_REFERENCE ID type in test_misc19(). This fails when running testhdf5 and skipping the reference test since the H5R package won't be initialized. H5I_REFERENCE will be going away soon and the ID type is unused so there's really no point to doing anything to test it. * Cleaned up test_misc13(), which is a basic test of userblock functionality, to not emit warnings due to the large global array. --- test/tmisc.c | 282 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 140 insertions(+), 142 deletions(-) diff --git a/test/tmisc.c b/test/tmisc.c index 22ae558..708ca9b 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -30,7 +30,6 @@ #include "testhdf5.h" #include "H5srcdir.h" #include "H5Dpkg.h" /* Datasets */ -#include "H5Iprivate.h" /* IDs, can be removed when H5I_REFERENCE is gone */ /* Definitions for misc. test #1 */ #define MISC1_FILE "tmisc1.h5" @@ -179,17 +178,12 @@ typedef struct #define MISC13_GROUP1_NAME "Group1" #define MISC13_GROUP2_NAME "Group2" #define MISC13_DTYPE_NAME "Datatype" -#define MISC13_RANK 2 -#define MISC13_DIM1 20 -#define MISC13_DIM2 30 +#define MISC13_RANK 1 +#define MISC13_DIM1 600 #define MISC13_CHUNK_DIM1 10 -#define MISC13_CHUNK_DIM2 15 #define MISC13_USERBLOCK_SIZE 512 #define MISC13_COPY_BUF_SIZE 4096 -unsigned m13_data[MISC13_DIM1][MISC13_DIM2]; /* Data to write to dataset */ -unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2]; /* Data read from dataset */ - /* Definitions for misc. test #14 */ #define MISC14_FILE "tmisc14.h5" #define MISC14_DSET1_NAME "Dataset1" @@ -2084,38 +2078,36 @@ test_misc12(void) /* Various routines for misc. 13 test */ static void -init_data(void) +misc13_init_data(unsigned *original_data) { - unsigned u,v; /* Local index variables */ + unsigned u; - for(u=0; u= 0); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_create_hdf_file() */ static void -insert_user_block(const char *old_name, const char *new_name,const char *str,size_t size) +misc13_insert_user_block(const char *old_name, const char *new_name, const char *str, size_t size) { - FILE *new_fp, *old_fp; /* Pointers to new & old files */ - void *user_block; /* Pointer to user block to write to file */ - void *copy_buf; /* Pointer to buffer for copying data */ + FILE *new_fp = NULL; /* Pointers to new & old files */ + FILE *old_fp = NULL; + void *user_block = NULL; /* Pointer to user block to write to file */ + void *copy_buf = NULL; /* Pointer to buffer for copying data */ size_t written; /* Amount of data written to new file */ size_t read_in; /* Amount of data read in from old file */ int ret; /* Generic status value */ @@ -2264,10 +2267,10 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz CHECK(user_block, NULL, "HDcalloc"); /* Copy in the user block data */ - HDmemcpy(user_block,str,strlen(str)); + HDmemcpy(user_block, str, strlen(str)); /* Open the new file */ - new_fp=HDfopen(new_name,"wb"); + new_fp = HDfopen(new_name,"wb"); CHECK(new_fp, NULL, "HDfopen"); /* Write the user block to the new file */ @@ -2275,7 +2278,7 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz VERIFY(written, size, "HDfwrite"); /* Open the old file */ - old_fp=HDfopen(old_name,"rb"); + old_fp = HDfopen(old_name,"rb"); CHECK(old_fp, NULL, "HDfopen"); /* Allocate space for the copy buffer */ @@ -2287,14 +2290,14 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz /* Write the data to the new file */ written = HDfwrite(copy_buf, (size_t)1, read_in, new_fp); VERIFY(written, read_in, "HDfwrite"); - } /* end while */ + } /* Close the old file */ - ret=HDfclose(old_fp); + ret = HDfclose(old_fp); VERIFY(ret, 0, "HDfclose"); /* Close the new file */ - ret=HDfclose(new_fp); + ret = HDfclose(new_fp); VERIFY(ret, 0, "HDfclose"); /* Free the copy buffer */ @@ -2302,81 +2305,84 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz /* Free the user block */ HDfree(user_block); -} + +} /* end misc13_insert_user_block() */ static void -verify_file(const char *name, hsize_t blk_size, unsigned check_new_data) +misc13_verify_file(const char *name, const unsigned *data, hsize_t userblock_size, + hbool_t check_for_new_dataset) { - hid_t fid; /* File ID */ - hid_t gid,gid2; /* Group IDs */ - hid_t tid; /* Datatype ID */ - hid_t fcpl; /* File creation property list ID */ - hsize_t userblock; /* Userblock size retrieved from FCPL */ - herr_t ret; /* Generic return value */ + hid_t fid = -1; /* File ID */ + hid_t gid1 = -1; /* Group IDs */ + hid_t gid2 = -1; /* Group IDs */ + hid_t tid = -1; /* Datatype ID */ + hid_t fcplid = -1; /* File creation property list ID */ + hsize_t ub_size_out; /* Userblock size retrieved from FCPL */ + herr_t ret; /* Generic return value */ /* Open the file */ - fid=H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT); + fid = H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Get the file's FCPL */ - fcpl=H5Fget_create_plist(fid); - CHECK(fcpl, FAIL, "H5Fget_create_plist"); + fcplid = H5Fget_create_plist(fid); + CHECK(fcplid, FAIL, "H5Fget_create_plist"); /* Get the user block size for the file */ - ret=H5Pget_userblock(fcpl,&userblock); + ret = H5Pget_userblock(fcplid, &ub_size_out); CHECK(ret, FAIL, "H5Pget_userblock"); /* Check the userblock size */ - VERIFY(userblock, blk_size, "H5Pget_userblock"); + VERIFY(userblock_size, ub_size_out, "H5Pget_userblock"); /* Close the FCPL */ - ret = H5Pclose(fcpl); + ret = H5Pclose(fcplid); CHECK(ret, FAIL, "H5Pclose"); /* Verify the contiguous dataset in the root group */ - verify_dataset(fid,MISC13_DSET1_NAME); + misc13_verify_dataset(fid, MISC13_DSET1_NAME, data); /* Verify the chunked dataset in the root group */ - verify_dataset(fid,MISC13_DSET2_NAME); + misc13_verify_dataset(fid, MISC13_DSET2_NAME, data); /* Verify the "new" contiguous dataset in the root group, if asked */ - if(check_new_data) - verify_dataset(fid,MISC13_DSET3_NAME); + if(check_for_new_dataset) + misc13_verify_dataset(fid, MISC13_DSET3_NAME, data); /* Open the named datatype in the root group */ tid = H5Topen2(fid, MISC13_DTYPE_NAME, H5P_DEFAULT); CHECK(tid, FAIL, "H5Topen2"); /* Verify the type is correct */ - VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal"); + VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal"); /* Close named datatype */ - ret=H5Tclose(tid); + ret = H5Tclose(tid); CHECK(ret, FAIL, "H5Tclose"); /* Open the first group */ - gid = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT); - CHECK(gid, FAIL, "H5Gopen2"); + gid1 = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT); + CHECK(gid1, FAIL, "H5Gopen2"); /* Verify the contiguous dataset in the first group */ - verify_dataset(gid,MISC13_DSET1_NAME); + misc13_verify_dataset(gid1, MISC13_DSET1_NAME, data); /* Verify the chunked dataset in the first group */ - verify_dataset(gid,MISC13_DSET2_NAME); + misc13_verify_dataset(gid1, MISC13_DSET2_NAME, data); /* Open the named datatype in the first group */ - tid = H5Topen2(gid,MISC13_DTYPE_NAME, H5P_DEFAULT); + tid = H5Topen2(gid1, MISC13_DTYPE_NAME, H5P_DEFAULT); CHECK(tid, FAIL, "H5Topen2"); /* Verify the type is correct */ - VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal"); + VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal"); /* Close named datatype */ - ret=H5Tclose(tid); + ret = H5Tclose(tid); CHECK(ret, FAIL, "H5Tclose"); /* Open the second group */ - gid2 = H5Gopen2(gid, MISC13_GROUP2_NAME, H5P_DEFAULT); + gid2 = H5Gopen2(gid1, MISC13_GROUP2_NAME, H5P_DEFAULT); CHECK(gid2, FAIL, "H5Gopen2"); /* Close the second group */ @@ -2384,31 +2390,33 @@ verify_file(const char *name, hsize_t blk_size, unsigned check_new_data) CHECK(ret, FAIL, "H5Gclose"); /* Close the first group */ - ret = H5Gclose(gid); + ret = H5Gclose(gid1); CHECK(ret, FAIL, "H5Gclose"); /* Close the file */ ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_verify_file() */ static void -add_to_new_file(const char *name) +misc13_add_to_new_file(const char *name, const unsigned *data) { - hid_t fid; /* File ID */ + hid_t fid = -1; /* File ID */ herr_t ret; /* Generic return value */ /* Open the file */ - fid=H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Create new contiguous dataset in root group */ - create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT); + misc13_create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT, data); /* Close the file */ ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_add_to_new_file() */ /**************************************************************** ** @@ -2419,26 +2427,44 @@ add_to_new_file(const char *name) static void test_misc13(void) { + unsigned *data = NULL; /* Data to write to dataset */ + hsize_t userblock_size; /* Correct size of userblock */ + hbool_t check_for_new_dataset; /* Whether to check for the post-userblock-creation dataset */ + + /* Create a data buffer for the datasets */ + data = (unsigned *)HDcalloc(MISC13_DIM1, sizeof(unsigned)); + CHECK(data, NULL, "HDcalloc"); + /* Initialize data to write */ - init_data(); + misc13_init_data(data); /* Create first file, with no user block */ - create_hdf_file(MISC13_FILE_1); + misc13_create_hdf_file(MISC13_FILE_1, data); /* Verify file contents are correct */ - verify_file(MISC13_FILE_1,(hsize_t)0,0); + userblock_size = 0; + check_for_new_dataset = FALSE; + misc13_verify_file(MISC13_FILE_1, data, userblock_size, check_for_new_dataset); /* Create a new file by inserting a user block in front of the first file */ - insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE); + misc13_insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE); /* Verify file contents are still correct */ - verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,0); + userblock_size = MISC13_USERBLOCK_SIZE; + check_for_new_dataset = FALSE; + misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset); /* Make certain we can modify the new file */ - add_to_new_file(MISC13_FILE_2); + misc13_add_to_new_file(MISC13_FILE_2, data); /* Verify file contents are still correct */ - verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,1); + userblock_size = MISC13_USERBLOCK_SIZE; + check_for_new_dataset = TRUE; + misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset); + + /* Free the dataset buffer */ + HDfree(data); + } /* end test_misc13() */ /**************************************************************** @@ -3448,34 +3474,6 @@ test_misc19(void) HDfree(vfd_cls); -/* Check H5I operations on references */ - - /* Reference IDs are not used by the library so there's no - * way of testing if incr/decr, etc. work. Instead, just - * do a quick smoke check to ensure that a couple of basic - * calls return sane values. - * - * H5I_REFERENCE has been declared deprecated with a tentative - * removal version of HDF5 1.12.0. - * - * Delete this entire block when H5I_REFERENCE no longer exists. - * - * The H5Iprivate.h header was included to support H5I_nmembers() - * so that can also probably be removed as well. - */ -{ - htri_t tf; /* Boolean generic return */ - int64_t num_members; /* Number of members in type */ - - tf = H5Itype_exists(H5I_REFERENCE); - VERIFY(tf, TRUE, "H5Itype_exists"); - - num_members = 999; - num_members = H5I_nmembers(H5I_REFERENCE); - VERIFY(num_members, 0, "H5Inmembers"); - -} /* end block */ - } /* end test_misc19() */ /**************************************************************** -- cgit v0.12 From 47a4cd816cff5e1223f66614e150777f9dca7af0 Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Wed, 5 Jul 2017 15:53:19 -0400 Subject: Commit changes needed for pull request --- testpar/t_bigio.c | 2097 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2097 insertions(+) create mode 100644 testpar/t_bigio.c diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c new file mode 100644 index 0000000..2f80e45 --- /dev/null +++ b/testpar/t_bigio.c @@ -0,0 +1,2097 @@ + +#include "hdf5.h" +#include "testphdf5.h" +#include "H5Dprivate.h" /* For Chunk tests */ + +// int TestVerbosity = VERBO_LO; /* Default Verbosity is Low */ + +/* Constants definitions */ +#define MAX_ERR_REPORT 10 /* Maximum number of errors reported */ + +/* Define some handy debugging shorthands, routines, ... */ +/* debugging tools */ + +#define MAINPROCESS (!mpi_rank) /* define process 0 as main process */ + +/* Constants definitions */ +#define RANK 2 + +#define IN_ORDER 1 +#define OUT_OF_ORDER 2 + +#define DATASET1 "DSET1" +#define DATASET2 "DSET2" +#define DATASET3 "DSET3" +#define DATASET4 "DSET4" +#define DATASET5 "DSET5" +#define DXFER_COLLECTIVE_IO 0x1 /* Collective IO*/ +#define DXFER_INDEPENDENT_IO 0x2 /* Independent IO collectively */ +#define DXFER_BIGCOUNT 536870916 + +#define HYPER 1 +#define POINT 2 +#define ALL 3 + +/* Dataset data type. Int's can be easily octo dumped. */ +typedef hsize_t B_DATATYPE; + +int facc_type = FACC_MPIO; /*Test file access type */ +int dxfer_coll_type = DXFER_COLLECTIVE_IO; +size_t bigcount = DXFER_BIGCOUNT; +char filename[20] = "bigio_test.h5"; +int nerrors = 0; +int mpi_size, mpi_rank; + +hsize_t space_dim1 = SPACE_DIM1 * 256; // 4096 +hsize_t space_dim2 = SPACE_DIM2; + +static void coll_chunktest(const char* filename, int chunk_factor, int select_factor, + int api_option, int file_selection, int mem_selection, int mode); +hid_t create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type); + +/* + * Setup the coordinates for point selection. + */ +static void +set_coords(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + size_t num_points, + hsize_t coords[], + int order) +{ + hsize_t i,j, k = 0, m ,n, s1 ,s2; + + if(OUT_OF_ORDER == order) + k = (num_points * RANK) - 1; + else if(IN_ORDER == order) + k = 0; + + s1 = start[0]; + s2 = start[1]; + + for(i = 0 ; i < count[0]; i++) + for(j = 0 ; j < count[1]; j++) + for(m = 0 ; m < block[0]; m++) + for(n = 0 ; n < block[1]; n++) + if(OUT_OF_ORDER == order) { + coords[k--] = s2 + (stride[1] * j) + n; + coords[k--] = s1 + (stride[0] * i) + m; + } + else if(IN_ORDER == order) { + coords[k++] = s1 + stride[0] * i + m; + coords[k++] = s2 + stride[1] * j + n; + } +} + +/* + * Fill the dataset with trivial data for testing. + * Assume dimension rank is 2 and data is stored contiguous. + */ +static void +fill_datasets(hsize_t start[], hsize_t block[], B_DATATYPE * dataset) +{ + B_DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* put some trivial data in the data_array */ + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + *dataptr = (B_DATATYPE)((i+start[0])*100 + (j+start[1]+1)); + dataptr++; + } + } +} + +/* + * Setup the coordinates for point selection. + */ +void point_set(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + size_t num_points, + hsize_t coords[], + int order) +{ + hsize_t i,j, k = 0, m ,n, s1 ,s2; + + HDcompile_assert(RANK == 2); + + if(OUT_OF_ORDER == order) + k = (num_points * RANK) - 1; + else if(IN_ORDER == order) + k = 0; + + s1 = start[0]; + s2 = start[1]; + + for(i = 0 ; i < count[0]; i++) + for(j = 0 ; j < count[1]; j++) + for(m = 0 ; m < block[0]; m++) + for(n = 0 ; n < block[1]; n++) + if(OUT_OF_ORDER == order) { + coords[k--] = s2 + (stride[1] * j) + n; + coords[k--] = s1 + (stride[0] * i) + m; + } + else if(IN_ORDER == order) { + coords[k++] = s1 + stride[0] * i + m; + coords[k++] = s2 + stride[1] * j + n; + } + + if(VERBOSE_MED) { + printf("start[]=(%lu, %lu), count[]=(%lu, %lu), stride[]=(%lu, %lu), block[]=(%lu, %lu), total datapoints=%lu\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1], + (unsigned long)(block[0] * block[1] * count[0] * count[1])); + k = 0; + for(i = 0; i < num_points ; i++) { + printf("(%d, %d)\n", (int)coords[k], (int)coords[k + 1]); + k += 2; + } + } +} + +/* + * Print the content of the dataset. + */ +static void +dataset_print(hsize_t start[], hsize_t block[], B_DATATYPE * dataset) +{ + B_DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* print the column heading */ + printf("%-8s", "Cols:"); + for (j=0; j < block[1]; j++){ + printf("%3lu ", (unsigned long)(start[1]+j)); + } + printf("\n"); + + /* print the slab data */ + for (i=0; i < block[0]; i++){ + printf("Row %2lu: ", (unsigned long)(i+start[0])); + for (j=0; j < block[1]; j++){ + printf("%llu ", *dataptr++); + } + printf("\n"); + } +} + + +/* + * Print the content of the dataset. + */ +static int +verify_data(hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[], B_DATATYPE *dataset, B_DATATYPE *original) +{ + hsize_t i, j; + int vrfyerrs; + + /* print it if VERBOSE_MED */ + if(VERBOSE_MED) { + printf("verify_data dumping:::\n"); + printf("start(%lu, %lu), count(%lu, %lu), stride(%lu, %lu), block(%lu, %lu)\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1]); + printf("original values:\n"); + dataset_print(start, block, original); + printf("compared values:\n"); + dataset_print(start, block, dataset); + } + + vrfyerrs = 0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(*dataset != *original){ + if(vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED){ + printf("Dataset Verify failed at [%lu][%lu](row %lu, col %lu): expect %llu, got %llu\n", + (unsigned long)i, (unsigned long)j, + (unsigned long)(i+start[0]), (unsigned long)(j+start[1]), + *(original), *(dataset)); + } + dataset++; + original++; + } + } + } + if(vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) + printf("[more errors ...]\n"); + if(vrfyerrs) + printf("%d errors found in verify_data\n", vrfyerrs); + return(vrfyerrs); +} + +/* Set up the selection */ +static void +ccslab_set(int mpi_rank, + int mpi_size, + hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + int mode) +{ + + switch (mode){ + + case BYROW_CONT: + /* Each process takes a slabs of rows. */ + block[0] = 1; + block[1] = 1; + stride[0] = 1; + stride[1] = 1; + count[0] = space_dim1; + count[1] = space_dim2; + start[0] = mpi_rank*count[0]; + start[1] = 0; + + break; + + case BYROW_DISCONT: + /* Each process takes several disjoint blocks. */ + block[0] = 1; + block[1] = 1; + stride[0] = 3; + stride[1] = 3; + count[0] = space_dim1/(stride[0]*block[0]); + count[1] = (space_dim2)/(stride[1]*block[1]); + start[0] = space_dim1*mpi_rank; + start[1] = 0; + + break; + + case BYROW_SELECTNONE: + /* Each process takes a slabs of rows, there are + no selections for the last process. */ + block[0] = 1; + block[1] = 1; + stride[0] = 1; + stride[1] = 1; + count[0] = ((mpi_rank >= MAX(1,(mpi_size-2)))?0:space_dim1); + count[1] = space_dim2; + start[0] = mpi_rank*count[0]; + start[1] = 0; + + break; + + case BYROW_SELECTUNBALANCE: + /* The first one-third of the number of processes only + select top half of the domain, The rest will select the bottom + half of the domain. */ + + block[0] = 1; + count[0] = 2; + stride[0] = space_dim1*mpi_size/4+1; + block[1] = space_dim2; + count[1] = 1; + start[1] = 0; + stride[1] = 1; + if((mpi_rank *3)<(mpi_size*2)) start[0] = mpi_rank; + else start[0] = 1 + space_dim1*mpi_size/2 + (mpi_rank-2*mpi_size/3); + break; + + case BYROW_SELECTINCHUNK: + /* Each process will only select one chunk */ + + block[0] = 1; + count[0] = 1; + start[0] = mpi_rank*space_dim1; + stride[0]= 1; + block[1] = space_dim2; + count[1] = 1; + stride[1]= 1; + start[1] = 0; + + break; + + default: + /* Unknown mode. Set it to cover the whole dataset. */ + block[0] = space_dim1*mpi_size; + block[1] = space_dim2; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = 0; + + break; + } + if (VERBOSE_MED){ + printf("start[]=(%lu,%lu), count[]=(%lu,%lu), stride[]=(%lu,%lu), block[]=(%lu,%lu), total datapoints=%lu\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1], + (unsigned long)(block[0]*block[1]*count[0]*count[1])); + } +} + + +/* + * Fill the dataset with trivial data for testing. + * Assume dimension rank is 2. + */ +static void +ccdataset_fill(hsize_t start[], + hsize_t stride[], + hsize_t count[], + hsize_t block[], + DATATYPE * dataset, + int mem_selection) +{ + DATATYPE *dataptr = dataset; + DATATYPE *tmptr; + hsize_t i,j,k1,k2,k=0; + /* put some trivial data in the data_array */ + tmptr = dataptr; + + /* assign the disjoint block (two-dimensional)data array value + through the pointer */ + + for (k1 = 0; k1 < count[0]; k1++) { + for(i = 0; i < block[0]; i++) { + for(k2 = 0; k2 < count[1]; k2++) { + for(j = 0;j < block[1]; j++) { + + if (ALL != mem_selection) { + dataptr = tmptr + ((start[0]+k1*stride[0]+i)*space_dim2+ + start[1]+k2*stride[1]+j); + } + else { + dataptr = tmptr + k; + k++; + } + + *dataptr = (DATATYPE)(k1+k2+i+j); + } + } + } + } +} + +/* + * Print the first block of the content of the dataset. + */ +static void +ccdataset_print(hsize_t start[], + hsize_t block[], + DATATYPE * dataset) + +{ + DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* print the column heading */ + printf("Print only the first block of the dataset\n"); + printf("%-8s", "Cols:"); + for (j=0; j < block[1]; j++){ + printf("%3lu ", (unsigned long)(start[1]+j)); + } + printf("\n"); + + /* print the slab data */ + for (i=0; i < block[0]; i++){ + printf("Row %2lu: ", (unsigned long)(i+start[0])); + for (j=0; j < block[1]; j++){ + printf("%03d ", *dataptr++); + } + printf("\n"); + } +} + +/* + * Print the content of the dataset. + */ +static int +ccdataset_vrfy(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + DATATYPE *dataset, + DATATYPE *original, + int mem_selection) +{ + hsize_t i, j,k1,k2,k=0; + int vrfyerrs; + DATATYPE *dataptr,*oriptr; + + /* print it if VERBOSE_MED */ + if (VERBOSE_MED) { + printf("dataset_vrfy dumping:::\n"); + printf("start(%lu, %lu), count(%lu, %lu), stride(%lu, %lu), block(%lu, %lu)\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1]); + printf("original values:\n"); + ccdataset_print(start, block, original); + printf("compared values:\n"); + ccdataset_print(start, block, dataset); + } + + vrfyerrs = 0; + + for (k1=0;k1 MAX_ERR_REPORT && !VERBOSE_MED) + printf("[more errors ...]\n"); + if (vrfyerrs) + printf("%d errors found in ccdataset_vrfy\n", vrfyerrs); + return(vrfyerrs); +} + +/* + * Example of using the parallel HDF5 library to create two datasets + * in one HDF5 file with collective parallel access support. + * The Datasets are of sizes (number-of-mpi-processes x dim0) x dim1. + * Each process controls only a slab of size dim0 x dim1 within each + * dataset. [Note: not so yet. Datasets are of sizes dim0xdim1 and + * each process controls a hyperslab within.] + */ + +static void +dataset_big_write(void) +{ + + hid_t xfer_plist; /* Dataset transfer properties list */ + hid_t sid; /* Dataspace ID */ + hid_t file_dataspace; /* File dataspace ID */ + hid_t mem_dataspace; /* memory dataspace ID */ + hid_t dataset; + hid_t datatype; /* Datatype ID */ + hsize_t dims[RANK]; /* dataset dim sizes */ + hsize_t start[RANK]; /* for hyperslab setting */ + hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ + hsize_t block[RANK]; /* for hyperslab setting */ + hsize_t *coords = NULL; + int i; + herr_t ret; /* Generic return value */ + hid_t fid; /* HDF5 file ID */ + hid_t acc_tpl; /* File access templates */ + hsize_t h; + size_t num_points; + B_DATATYPE * wdata; + + + /* allocate memory for data buffer */ + wdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + /* setup file access template */ + acc_tpl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "H5P_FILE_ACCESS"); + H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* create the file collectively */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + VRFY((fid >= 0), "H5Fcreate succeeded"); + + /* Release file-access template */ + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), ""); + + + /* Each process takes a slabs of rows. */ + printf("\nTesting Dataset1 write by ROW\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET1, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill the local slab with some trivial data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* write data collectively */ + MESG("writeAll by Row"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + + /* Each process takes a slabs of cols. */ + printf("\nTesting Dataset2 write by COL\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET2, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]; + block[1] = dims[1]/mpi_size; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank*block[1]; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill the local slab with some trivial data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* write data collectively */ + MESG("writeAll by Col"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + + /* ALL selection */ + printf("\nTesting Dataset3 write select ALL proc 0, NONE others\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = 1; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET3, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + if(MAINPROCESS) { + ret = H5Sselect_all(file_dataspace); + VRFY((ret >= 0), "H5Sset_all succeeded"); + } + else { + ret = H5Sselect_none(file_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + if(!MAINPROCESS) { + ret = H5Sselect_none(mem_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* fill the local slab with some trivial data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* write data collectively */ + MESG("writeAll by process 0"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + /* Point selection */ + printf("\nTesting Dataset4 write point selection\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size * 4; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET4, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]/2; + block[1] = 2; + stride[0] = dims[0]/2; + stride[1] = 2; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = dims[1]/mpi_size * mpi_rank; + + num_points = bigcount; + + coords = (hsize_t *)malloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + + set_coords (start, count, stride, block, num_points, coords, IN_ORDER); + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((ret >= 0), "H5Sselect_elements succeeded"); + + if(coords) free(coords); + + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (1, &bigcount, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + /* Irregular selection */ + /* Need larger memory for data buffer */ + free(wdata); +#if 0 + wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + printf("\nTesting Dataset5 write irregular selection\n"); + /* Create a large dataset */ + dims[0] = bigcount/6; + dims[1] = mpi_size * 4; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET5, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + /* first select 1 col in this procs splice */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank * 4; + + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + + dims[1] = 4; + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* select every other row in the process splice and OR it with + the col selection to create an irregular selection */ + for(h=0 ; h= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + } + printf("Setting up for collective transfer\n"); + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* fill the local slab with some trivial data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + free(wdata); +#endif + H5Fclose(fid); +} + +/* + * Example of using the parallel HDF5 library to read two datasets + * in one HDF5 file with collective parallel access support. + * The Datasets are of sizes (number-of-mpi-processes x dim0) x dim1. + * Each process controls only a slab of size dim0 x dim1 within each + * dataset. [Note: not so yet. Datasets are of sizes dim0xdim1 and + * each process controls a hyperslab within.] + */ + +static void +dataset_big_read(void) +{ + hid_t fid; /* HDF5 file ID */ + hid_t acc_tpl; /* File access templates */ + hid_t xfer_plist; /* Dataset transfer properties list */ + hid_t file_dataspace; /* File dataspace ID */ + hid_t mem_dataspace; /* memory dataspace ID */ + hid_t dataset; + B_DATATYPE *rdata = NULL; /* data buffer */ + B_DATATYPE *wdata = NULL; /* expected data buffer */ + hsize_t dims[RANK]; /* dataset dim sizes */ + hsize_t start[RANK]; /* for hyperslab setting */ + hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ + hsize_t block[RANK]; /* for hyperslab setting */ + int i,j,k; + hsize_t h; + size_t num_points; + hsize_t *coords = NULL; + herr_t ret; /* Generic return value */ + + /* allocate memory for data buffer */ + rdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((rdata != NULL), "rdata malloc succeeded"); + wdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + + /* setup file access template */ + acc_tpl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "H5P_FILE_ACCESS"); + H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* open the file collectively */ + fid=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl); + VRFY((fid >= 0), "H5Fopen succeeded"); + + /* Release file-access template */ + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), ""); + + + printf("\nRead Testing Dataset1 by COL\n"); + dataset = H5Dopen2(fid, DATASET1, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size; + /* Each process takes a slabs of cols. */ + block[0] = dims[0]; + block[1] = dims[1]/mpi_size; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank*block[1]; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill dataset with test data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + printf("\nRead Testing Dataset2 by ROW\n"); + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + dataset = H5Dopen2(fid, DATASET2, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size; + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill dataset with test data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset2 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + printf("\nRead Testing Dataset3 read select ALL proc 0, NONE others\n"); + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + dataset = H5Dopen2(fid, DATASET3, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = 1; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + if(MAINPROCESS) { + ret = H5Sselect_all(file_dataspace); + VRFY((ret >= 0), "H5Sset_all succeeded"); + } + else { + ret = H5Sselect_none(file_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + if(!MAINPROCESS) { + ret = H5Sselect_none(mem_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* fill dataset with test data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset3 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + if(MAINPROCESS) { + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + } + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + printf("\nRead Testing Dataset4 with Point selection\n"); + dataset = H5Dopen2(fid, DATASET4, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size * 4; + + block[0] = dims[0]/2; + block[1] = 2; + stride[0] = dims[0]/2; + stride[1] = 2; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = dims[1]/mpi_size * mpi_rank; + + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + num_points = bigcount; + + coords = (hsize_t *)malloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + + set_coords (start, count, stride, block, num_points, coords, IN_ORDER); + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((ret >= 0), "H5Sselect_elements succeeded"); + + if(coords) free(coords); + + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (1, &bigcount, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + printf("\nRead Testing Dataset5 with Irregular selection\n"); + /* Need larger memory for data buffer */ + free(wdata); + free(rdata); +#if 0 + wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + rdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((rdata != NULL), "rdata malloc succeeded"); + + dataset = H5Dopen2(fid, DATASET5, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size * 4; + + /* first select 1 col in this proc splice */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank * 4; + + /* get file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + + /* create a memory dataspace */ + dims[1] = 4; + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* select every other row in the process splice and OR it with + the col selection to create an irregular selection */ + for(h=0 ; h= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + //fprintf(stderr, "%d: %d - %d\n", mpi_rank, (int)h, (int)H5Sget_select_npoints(mem_dataspace)); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + /* fill dataset with test data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + + + /* verify the read data with original expected data */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = 0; + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + for(h=0 ; h= 0), "H5Dclose1 succeeded"); + + H5Fclose(fid); + + /* release data buffers */ + if(rdata) free(rdata); + if(wdata) free(wdata); +#endif +} /* dataset_large_readAll */ + + +/* + * Create the appropriate File access property list + */ +hid_t +create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type) +{ + hid_t ret_pl = -1; + herr_t ret; /* generic return value */ + int mpi_rank; /* mpi variables */ + + /* need the rank for error checking macros */ + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + ret_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((ret_pl >= 0), "H5P_FILE_ACCESS"); + + if (l_facc_type == FACC_DEFAULT) + return (ret_pl); + + if (l_facc_type == FACC_MPIO){ + /* set Parallel access with communicator */ + ret = H5Pset_fapl_mpio(ret_pl, comm, info); + VRFY((ret >= 0), ""); + ret = H5Pset_all_coll_metadata_ops(ret_pl, TRUE); + VRFY((ret >= 0), ""); + ret = H5Pset_coll_metadata_write(ret_pl, TRUE); + VRFY((ret >= 0), ""); + return(ret_pl); + } + + if (l_facc_type == (FACC_MPIO | FACC_SPLIT)){ + hid_t mpio_pl; + + mpio_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((mpio_pl >= 0), ""); + /* set Parallel access with communicator */ + ret = H5Pset_fapl_mpio(mpio_pl, comm, info); + VRFY((ret >= 0), ""); + + /* setup file access template */ + ret_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((ret_pl >= 0), ""); + /* set Parallel access with communicator */ + ret = H5Pset_fapl_split(ret_pl, ".meta", mpio_pl, ".raw", mpio_pl); + VRFY((ret >= 0), "H5Pset_fapl_split succeeded"); + H5Pclose(mpio_pl); + return(ret_pl); + } + + /* unknown file access types */ + return (ret_pl); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk1 + * + * Purpose: Wrapper to test the collective chunk IO for regular JOINT + selection with a single chunk + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +/* ------------------------------------------------------------------------ + * Descriptions for the selection: One big singluar selection inside one chunk + * Two dimensions, + * + * dim1 = space_dim1(5760)*mpi_size + * dim2 = space_dim2(3) + * chunk_dim1 = dim1 + * chunk_dim2 = dim2 + * block = 1 for all dimensions + * stride = 1 for all dimensions + * count0 = space_dim1(5760) + * count1 = space_dim2(3) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * ------------------------------------------------------------------------ + */ + +void +coll_chunk1(void) +{ + if (MAINPROCESS) + printf("coll_chunk1\n"); + + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk2 + * + * Purpose: Wrapper to test the collective chunk IO for regular DISJOINT + selection with a single chunk + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + + /* ------------------------------------------------------------------------ + * Descriptions for the selection: many disjoint selections inside one chunk + * Two dimensions, + * + * dim1 = space_dim1*mpi_size(5760) + * dim2 = space_dim2(3) + * chunk_dim1 = dim1 + * chunk_dim2 = dim2 + * block = 1 for all dimensions + * stride = 3 for all dimensions + * count0 = space_dim1/stride0(5760/3) + * count1 = space_dim2/stride(3/3 = 1) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * + * ------------------------------------------------------------------------ + */ +void +coll_chunk2(void) +{ + if (MAINPROCESS) + printf("coll_chunk2\n"); + + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk3 + * + * Purpose: Wrapper to test the collective chunk IO for regular JOINT + selection with at least number of 2*mpi_size chunks + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +/* ------------------------------------------------------------------------ + * Descriptions for the selection: one singular selection accross many chunks + * Two dimensions, Num of chunks = 2* mpi_size + * + * dim1 = space_dim1*mpi_size + * dim2 = space_dim2(3) + * chunk_dim1 = space_dim1 + * chunk_dim2 = dim2/2 + * block = 1 for all dimensions + * stride = 1 for all dimensions + * count0 = space_dim1 + * count1 = space_dim2(3) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * + * ------------------------------------------------------------------------ + */ + +void +coll_chunk3(void) +{ + if (MAINPROCESS) + printf("coll_chunk3\n"); + + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +//------------------------------------------------------------------------- +// Borrowed/Modified (slightly) from t_coll_chunk.c +/*------------------------------------------------------------------------- + * Function: coll_chunktest + * + * Purpose: The real testing routine for regular selection of collective + chunking storage + testing both write and read, + If anything fails, it may be read or write. There is no + separation test between read and write. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Modifications: + * Remove invalid temporary property checkings for API_LINK_HARD and + * API_LINK_TRUE cases. + * Programmer: Jonathan Kim + * Date: 2012-10-10 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static void +coll_chunktest(const char* filename, + int chunk_factor, + int select_factor, + int api_option, + int file_selection, + int mem_selection, + int mode) +{ + hid_t file, dataset, file_dataspace, mem_dataspace; + hid_t acc_plist,xfer_plist,crp_plist; + + hsize_t dims[RANK], chunk_dims[RANK]; + int* data_array1 = NULL; + int* data_origin1 = NULL; + + hsize_t start[RANK],count[RANK],stride[RANK],block[RANK]; + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + unsigned prop_value; +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + herr_t status; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + size_t num_points; /* for point selection */ + hsize_t *coords = NULL; /* for point selection */ + hsize_t current_dims; /* for point selection */ + int i; + + /* Create the data space */ + + acc_plist = create_faccess_plist(comm,info,facc_type); + VRFY((acc_plist >= 0),""); + + file = H5Fcreate(filename,H5F_ACC_TRUNC,H5P_DEFAULT,acc_plist); + VRFY((file >= 0),"H5Fcreate succeeded"); + + status = H5Pclose(acc_plist); + VRFY((status >= 0),""); + + /* setup dimensionality object */ + dims[0] = space_dim1*mpi_size; + dims[1] = space_dim2; + + /* allocate memory for data buffer */ + data_array1 = (int *)HDmalloc(dims[0] * dims[1] * sizeof(int)); + VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); + + /* set up dimensions of the slab this process accesses */ + ccslab_set(mpi_rank, mpi_size, start, count, stride, block, select_factor); + + /* set up the coords array selection */ + num_points = block[0] * block[1] * count[0] * count[1]; + coords = (hsize_t *)HDmalloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + point_set(start, count, stride, block, num_points, coords, mode); + + file_dataspace = H5Screate_simple(2, dims, NULL); + VRFY((file_dataspace >= 0), "file dataspace created succeeded"); + + if(ALL != mem_selection) { + mem_dataspace = H5Screate_simple(2, dims, NULL); + VRFY((mem_dataspace >= 0), "mem dataspace created succeeded"); + } + else { + current_dims = num_points; + mem_dataspace = H5Screate_simple (1, ¤t_dims, NULL); + VRFY((mem_dataspace >= 0), "mem_dataspace create succeeded"); + } + + crp_plist = H5Pcreate(H5P_DATASET_CREATE); + VRFY((crp_plist >= 0),""); + + /* Set up chunk information. */ + chunk_dims[0] = dims[0]/chunk_factor; + + /* to decrease the testing time, maintain bigger chunk size */ + (chunk_factor == 1) ? (chunk_dims[1] = space_dim2) : (chunk_dims[1] = space_dim2/2); + status = H5Pset_chunk(crp_plist, 2, chunk_dims); + VRFY((status >= 0),"chunk creation property list succeeded"); + + dataset = H5Dcreate2(file, DSET_COLLECTIVE_CHUNK_NAME, H5T_NATIVE_INT, + file_dataspace, H5P_DEFAULT, crp_plist, H5P_DEFAULT); + VRFY((dataset >= 0),"dataset created succeeded"); + + status = H5Pclose(crp_plist); + VRFY((status >= 0), ""); + + /*put some trivial data in the data array */ + ccdataset_fill(start, stride, count,block, data_array1, mem_selection); + + MESG("data_array initialized"); + + switch (file_selection) { + case HYPER: + status = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(file_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(file_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + switch (mem_selection) { + case HYPER: + status = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(mem_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(mem_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + /* set up the collective transfer property list */ + xfer_plist = H5Pcreate(H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + + status = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((status>= 0),"MPIO collective transfer property succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + status = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); + VRFY((status>= 0),"set independent IO collectively succeeded"); + } + + switch(api_option){ + case API_LINK_HARD: + status = H5Pset_dxpl_mpio_chunk_opt(xfer_plist,H5FD_MPIO_CHUNK_ONE_IO); + VRFY((status>= 0),"collective chunk optimization succeeded"); + break; + + case API_MULTI_HARD: + status = H5Pset_dxpl_mpio_chunk_opt(xfer_plist,H5FD_MPIO_CHUNK_MULTI_IO); + VRFY((status>= 0),"collective chunk optimization succeeded "); + break; + + case API_LINK_TRUE: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,2); + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + break; + + case API_LINK_FALSE: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,6); + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + break; + + case API_MULTI_COLL: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,8);/* make sure it is using multi-chunk IO */ + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + status = H5Pset_dxpl_mpio_chunk_opt_ratio(xfer_plist,50); + VRFY((status>= 0),"collective chunk optimization set chunk ratio succeeded"); + break; + + case API_MULTI_IND: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,8);/* make sure it is using multi-chunk IO */ + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + status = H5Pset_dxpl_mpio_chunk_opt_ratio(xfer_plist,100); + VRFY((status>= 0),"collective chunk optimization set chunk ratio succeeded"); + break; + + default: + ; + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if(facc_type == FACC_MPIO) { + switch(api_option) { + case API_LINK_HARD: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_HARD_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_HARD: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_HARD_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_LINK_TRUE: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_NUM_TRUE_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_LINK_FALSE: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_NUM_FALSE_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_COLL: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_RATIO_COLL_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_IND: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + default: + ; + } + } +#endif + + /* write data collectively */ + status = H5Dwrite(dataset, H5T_NATIVE_INT, mem_dataspace, file_dataspace, + xfer_plist, data_array1); + VRFY((status >= 0),"dataset write succeeded"); + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if(facc_type == FACC_MPIO) { + switch(api_option){ + case API_LINK_HARD: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_HARD_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK COLLECTIVE IO directly succeeded"); + break; + + case API_MULTI_HARD: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_HARD_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK COLLECTIVE IO optimization succeeded"); + break; + + case API_LINK_TRUE: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_NUM_TRUE_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK COLLECTIVE IO succeeded"); + break; + + case API_LINK_FALSE: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_NUM_FALSE_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK IO transferring to multi-chunk IO succeeded"); + break; + + case API_MULTI_COLL: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_RATIO_COLL_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK COLLECTIVE IO with optimization succeeded"); + break; + + case API_MULTI_IND: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK IO transferring to independent IO succeeded"); + break; + + default: + ; + } + } +#endif + + status = H5Dclose(dataset); + VRFY((status >= 0),""); + + status = H5Pclose(xfer_plist); + VRFY((status >= 0),"property list closed"); + + status = H5Sclose(file_dataspace); + VRFY((status >= 0),""); + + status = H5Sclose(mem_dataspace); + VRFY((status >= 0),""); + + + status = H5Fclose(file); + VRFY((status >= 0),""); + + if (data_array1) HDfree(data_array1); + + /* Use collective read to verify the correctness of collective write. */ + + /* allocate memory for data buffer */ + data_array1 = (int *)HDmalloc(dims[0]*dims[1]*sizeof(int)); + VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); + + /* allocate memory for data buffer */ + data_origin1 = (int *)HDmalloc(dims[0]*dims[1]*sizeof(int)); + VRFY((data_origin1 != NULL), "data_origin1 malloc succeeded"); + + acc_plist = create_faccess_plist(comm, info, facc_type); + VRFY((acc_plist >= 0),"MPIO creation property list succeeded"); + + file = H5Fopen(filename,H5F_ACC_RDONLY,acc_plist); + VRFY((file >= 0),"H5Fcreate succeeded"); + + status = H5Pclose(acc_plist); + VRFY((status >= 0),""); + + /* open the collective dataset*/ + dataset = H5Dopen2(file, DSET_COLLECTIVE_CHUNK_NAME, H5P_DEFAULT); + VRFY((dataset >= 0), ""); + + /* set up dimensions of the slab this process accesses */ + ccslab_set(mpi_rank, mpi_size, start, count, stride, block, select_factor); + + /* obtain the file and mem dataspace*/ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), ""); + + if (ALL != mem_selection) { + mem_dataspace = H5Dget_space (dataset); + VRFY((mem_dataspace >= 0), ""); + } + else { + current_dims = num_points; + mem_dataspace = H5Screate_simple (1, ¤t_dims, NULL); + VRFY((mem_dataspace >= 0), "mem_dataspace create succeeded"); + } + + switch (file_selection) { + case HYPER: + status = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(file_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(file_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + switch (mem_selection) { + case HYPER: + status = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(mem_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(mem_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + /* fill dataset with test data */ + ccdataset_fill(start, stride,count,block, data_origin1, mem_selection); + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0),""); + + status = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((status>= 0),"MPIO collective transfer property succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + status = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((status>= 0),"set independent IO collectively succeeded"); + } + + status = H5Dread(dataset, H5T_NATIVE_INT, mem_dataspace, file_dataspace, + xfer_plist, data_array1); + VRFY((status >=0),"dataset read succeeded"); + + /* verify the read data with original expected data */ + status = ccdataset_vrfy(start, count, stride, block, data_array1, data_origin1, mem_selection); + if (status) nerrors++; + + status = H5Pclose(xfer_plist); + VRFY((status >= 0),"property list closed"); + + /* close dataset collectively */ + status=H5Dclose(dataset); + VRFY((status >= 0), "H5Dclose"); + + /* release all IDs created */ + status = H5Sclose(file_dataspace); + VRFY((status >= 0),"H5Sclose"); + + status = H5Sclose(mem_dataspace); + VRFY((status >= 0),"H5Sclose"); + + /* close the file collectively */ + status = H5Fclose(file); + VRFY((status >= 0),"H5Fclose"); + + /* release data buffers */ + if(coords) HDfree(coords); + if(data_array1) HDfree(data_array1); + if(data_origin1) HDfree(data_origin1); + +} + + + + + +int main(int argc, char **argv) +{ + + hsize_t newsize = 1048576; + hsize_t oldsize = H5S_mpio_set_bigio_count(newsize); + if (newsize != oldsize) { + bigcount = newsize * 2; + } + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + + dataset_big_write(); + MPI_Barrier(MPI_COMM_WORLD); + + dataset_big_read(); + MPI_Barrier(MPI_COMM_WORLD); + + oldsize = H5S_mpio_set_bigio_count(16384); + + coll_chunk1(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk2(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk3(); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Finalize(); + + return 0; +} + -- cgit v0.12 From 64d33e5e6e4b4270a3982c1be384cb41a0aa4c3b Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Wed, 5 Jul 2017 16:19:57 -0400 Subject: Commited changes to the development branch here to allow a pull request to be published --- src/H5Smpio.c | 504 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 458 insertions(+), 46 deletions(-) diff --git a/src/H5Smpio.c b/src/H5Smpio.c index c24c455..7319a80 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -33,7 +33,7 @@ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ #include "H5Spkg.h" /* Dataspaces */ -#include "H5VMprivate.h" /* Vector and array functions */ +#include "H5VMprivate.h" /* Vector and array functions */ #ifdef H5_HAVE_PARALLEL @@ -55,9 +55,42 @@ static herr_t H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size, MPI_Datatype *new_type, int *count, hbool_t *is_derived_type); static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span, const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size); +static herr_t H5S_mpio_create_large_type (hsize_t, MPI_Aint, MPI_Datatype , MPI_Datatype *); + #define H5S_MPIO_INITIAL_ALLOC_COUNT 256 +#define TWO_GIG_LIMIT 2147483648 + +#ifndef H5S_MAX_MPI_COUNT +#define H5S_MAX_MPI_COUNT 536870911 /* (2^29)-1 */ +#endif + +static hsize_t bigio_count = H5S_MAX_MPI_COUNT; + +/*------------------------------------------------------------------------- + * Function: H5S_mpio_set_bigio_count + * + * Purpose: Allow us to programatically change the switch point + * when we utilize derived datatypes. This is of + * particular interest for allowing nightly testing + * + * Return: the current/previous value of bigio_count. + * + * Programmer: Richard Warren, March 10, 2017 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5S_mpio_set_bigio_count(hsize_t new_count) +{ + hsize_t orig_count = bigio_count; + if ((new_count > 0) && (new_count < TWO_GIG_LIMIT)) { + bigio_count = new_count; + } + return orig_count; +} + /*------------------------------------------------------------------------- * Function: H5S_mpio_all_type @@ -72,6 +105,11 @@ static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span, * *is_derived_type 0 if MPI primitive type, 1 if derived * * Programmer: rky 980813 + * Modifications: + * Mohamad Chaarawi + * Adding support for large datatypes (beyond the limit of a + * 32 bit integer. + * * *------------------------------------------------------------------------- */ @@ -95,11 +133,22 @@ H5S_mpio_all_type(const H5S_t *space, size_t elmt_size, H5_CHECKED_ASSIGN(nelmts, hsize_t, snelmts, hssize_t); total_bytes = (hsize_t)elmt_size * nelmts; - - /* fill in the return values */ - *new_type = MPI_BYTE; - H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t); - *is_derived_type = FALSE; + /* Verify that the size can be expressed as a 32 bit integer */ + if(bigio_count >= total_bytes) { + /* fill in the return values */ + *new_type = MPI_BYTE; + H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t); + *is_derived_type = FALSE; + } + else { + /* Create a LARGE derived datatype for this transfer */ + if (H5S_mpio_create_large_type (total_bytes, 0, MPI_BYTE, new_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large datatype from the all selection") + } + *count = 1; + *is_derived_type = TRUE; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -167,27 +216,103 @@ H5S_mpio_create_point_datatype (size_t elmt_size, hsize_t num_points, HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) elmt_type_created = TRUE; + /* Check whether standard or BIGIO processing will be employeed */ + if(bigio_count >= num_points) { #if MPI_VERSION >= 3 - /* Create an MPI datatype for the whole point selection */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) + /* Create an MPI datatype for the whole point selection */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) #else - /* Allocate block sizes for MPI datatype call */ - if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points))) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + /* Allocate block sizes for MPI datatype call */ + if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") - for(u = 0; u < num_points; u++) - blocks[u] = 1; + for(u = 0; u < num_points; u++) + blocks[u] = 1; - /* Create an MPI datatype for the whole point selection */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) + /* Create an MPI datatype for the whole point selection */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) #endif - /* Commit MPI datatype for later use */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + /* Commit MPI datatype for later use */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + else { + /* use LARGE_DATATYPE:: + * We'll create an hindexed_block type for every 2G point count and then combine + * those and any remaining points into a single large datatype. + */ + int total_types, i; + int remaining_points; + int num_big_types; + hsize_t leftover; + + int *inner_blocks; + MPI_Aint *inner_disps; + MPI_Datatype *inner_types = NULL; + + /* Calculate how many Big MPI datatypes are needed to represent the buffer */ + num_big_types = (int)(num_points/bigio_count); + + leftover = (hsize_t)num_points - (hsize_t)num_big_types * (hsize_t)bigio_count; + H5_CHECKED_ASSIGN(remaining_points, int, leftover, hsize_t); + + total_types = (int)(remaining_points) ? (num_big_types + 1) : num_big_types; + + /* Allocate array if MPI derived types needed */ + if(NULL == (inner_types = (MPI_Datatype *)H5MM_malloc((sizeof(MPI_Datatype) * (size_t)total_types)))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + if(NULL == (inner_blocks = (int *)H5MM_malloc(sizeof(int) * (size_t)total_types))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + if(NULL == (inner_disps = (MPI_Aint *)H5MM_malloc(sizeof(MPI_Aint) * (size_t)total_types))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + for(i=0 ; i= elmt_size) { + /* Use a single MPI datatype that has a 32 bit size */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + else { + /* Create the compound datatype for this operation (> 2GB) */ + if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &inner_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't ccreate a large inner datatype in hyper selection") + } + } /******************************************************* * Construct the type by walking the hyperslab dims @@ -645,30 +790,93 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size, *******************************************************/ for(i = ((int)rank) - 1; i >= 0; --i) { #ifdef H5S_DEBUG - if(H5DEBUG(S)) - HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n" - "start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n", - FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]); + if(H5DEBUG(S)) + HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n" + "start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n", + FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]); #endif #ifdef H5S_DEBUG - if(H5DEBUG(S)) - HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i); + if(H5DEBUG(S)) + HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i); #endif /**************************************** * Build vector type of the selection. ****************************************/ - mpi_code = MPI_Type_vector((int)(d[i].count), /* count */ - (int)(d[i].block), /* blocklength */ - (int)(d[i].strid), /* stride */ - inner_type, /* old type */ - &outer_type); /* new type */ + if (bigio_count >= d[i].count && + bigio_count >= d[i].block && + bigio_count >= d[i].strid) { + + /* All the parameters fit into 32 bit integers so create the vector type normally */ + mpi_code = MPI_Type_vector((int)(d[i].count), /* count */ + (int)(d[i].block), /* blocklength */ + (int)(d[i].strid), /* stride */ + inner_type, /* old type */ + &outer_type); /* new type */ + + MPI_Type_free(&inner_type); + if(mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code) + } + else { + /* Things get a bit more complicated and require LARGE_DATATYPE processing + * There are two MPI datatypes that need to be created: + * 1) an internal contiguous block; and + * 2) a collection of elements where an element is a contiguous block(1). + * Remember that the input arguments to the MPI-IO functions use integer + * values to represent element counts. We ARE allowed however, in the + * more recent MPI implementations to use constructed datatypes whereby + * the total number of bytes in a transfer could be : + * (2GB-1)number_of_blocks * the_datatype_extent. + */ + + MPI_Aint stride_in_bytes, inner_extent; + MPI_Datatype block_type; + + /* create a contiguous datatype inner_type x number of BLOCKS. + * Again we need to check that the number of BLOCKS can fit into + * a 32 bit integer */ + if (bigio_count < d[i].block) { + if (H5S_mpio_create_large_type(d[i].block, 0, inner_type, + &block_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't ccreate a large block datatype in hyper selection") + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)d[i].block, + inner_type, + &block_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } - MPI_Type_free(&inner_type); - if(mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code) + MPI_Type_extent (inner_type, &inner_extent); + stride_in_bytes = inner_extent * (MPI_Aint)d[i].strid; - /**************************************** + /* If the element count is larger than what a 32 bit integer can hold, + * we call the large type creation function to handle that + */ + if (bigio_count < d[i].count) { + if (H5S_mpio_create_large_type (d[i].count, stride_in_bytes, block_type, + &outer_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large outer datatype in hyper selection") + } + } + /* otherwise a regular create_hvector will do */ + else { + mpi_code = MPI_Type_create_hvector((int)d[i].count, /* count */ + 1, /* blocklength */ + stride_in_bytes, /* stride in bytes*/ + block_type, /* old type */ + &outer_type); /* new type */ + if(MPI_SUCCESS != mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + MPI_Type_free(&block_type); + MPI_Type_free(&inner_type); + } + /**************************************** * Then build the dimension type as (start, vector type, xtent). ****************************************/ /* calculate start and extent values of this dimension */ @@ -752,6 +960,10 @@ done: * * Programmer: kyang * + * Modifications: + * Mohamad Chaarawi + * Adding support for large datatypes (beyond the limit of a + * 32 bit integer. *------------------------------------------------------------------------- */ static herr_t @@ -774,8 +986,17 @@ H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size, HDassert(space->select.sel_info.hslab->span_lst->head); /* Create the base type for an element */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + if (bigio_count >= elmt_size) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &elmt_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large element datatype in span_hyper selection") + } + } elmt_type_is_derived = TRUE; /* Compute 'down' sizes for each dimension */ @@ -821,14 +1042,15 @@ static herr_t H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span, const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size) { - size_t alloc_count; /* Number of span tree nodes allocated at this level */ - size_t outercount; /* Number of span tree nodes at this level */ + size_t alloc_count = 0; /* Number of span tree nodes allocated at this level */ + size_t outercount = 0; /* Number of span tree nodes at this level */ MPI_Datatype *inner_type = NULL; hbool_t inner_types_freed = FALSE; /* Whether the inner_type MPI datatypes have been freed */ hbool_t span_type_valid = FALSE; /* Whether the span_type MPI datatypes is valid */ + hbool_t large_block = FALSE; /* Wether the block length is larger than 32 bit integer */ int *blocklen = NULL; MPI_Aint *disp = NULL; - H5S_hyper_span_t *tspan; /* Temporary pointer to span tree node */ + H5S_hyper_span_t *tspan = NULL; /* Temporary pointer to span tree node */ int mpi_code; /* MPI return status code */ herr_t ret_value = SUCCEED; /* Return value */ @@ -870,14 +1092,70 @@ H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span, disp[outercount] = (MPI_Aint)elmt_size * tspan->low; H5_CHECK_OVERFLOW(tspan->nelem, hsize_t, int) blocklen[outercount] = (int)tspan->nelem; - tspan = tspan->next; + + if (bigio_count < blocklen[outercount]) { + large_block = TRUE; /* at least one block type is large, so set this flag to true */ + } + outercount++; } /* end while */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - span_type_valid = TRUE; + /* Everything fits into integers, so cast them and use hindexed */ + if (bigio_count >= outercount && large_block == FALSE) { + + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + span_type_valid = TRUE; + } + else { /* LARGE_DATATYPE:: Something doesn't fit into a 32 bit integer */ + size_t i; + + for (i=0 ; i bigio_count) { + if (H5S_mpio_create_large_type (blocklen[i], 0, *elmt_type, &temp_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large element datatype in span_hyper selection") + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)blocklen[i], + *elmt_type, + &temp_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + + /* combine the current datatype that is created with this current block type */ + if (0 == i) { /* first iteration, there is no combined datatype yet */ + *span_type = temp_type; + } + else { + int bl[2] = {1,1}; + MPI_Aint ds[2] = {disp[i-1],disp[i]}; + MPI_Datatype dt[2] = {*span_type, temp_type}; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_struct (2, /* count */ + bl, /* blocklength */ + ds, /* stride in bytes*/ + dt, /* old type */ + &outer_type))){ /* new type */ + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + } + *span_type = outer_type; + } + + if (outer_type != MPI_DATATYPE_NULL) + MPI_Type_free(&outer_type); + /* temp_type shouldn't be freed here... + * Note that we have simply copied it above (not MPI_Type_dup) + * into the 'span_type' argument of the caller. + * The caller needs to deal with it there! + */ + } + } /* end (LARGE_DATATYPE::) */ + } /* end if */ else { size_t u; /* Local index variable */ @@ -1091,5 +1369,139 @@ H5S_mpio_space_type(const H5S_t *space, size_t elmt_size, MPI_Datatype *new_type done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_mpio_space_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5S_mpio_create_large_type + * + * Purpose: Create a large datatype of size larger than what a 32 bit integer + * can hold. + * + * Return: non-negative on success, negative on failure. + * + * *new_type the new datatype created + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t H5S_mpio_create_large_type (hsize_t num_elements, + MPI_Aint stride_bytes, + MPI_Datatype old_type, + MPI_Datatype *new_type) +{ + int num_big_types; /* num times the 2G datatype will be repeated */ + int remaining_bytes; /* the number of bytes left that can be held in an int value */ + hsize_t leftover; + int block_len[2]; + int mpi_code; /* MPI return code */ + MPI_Datatype inner_type, outer_type, leftover_type, type[2]; + MPI_Aint disp[2], old_extent; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Calculate how many Big MPI datatypes are needed to represent the buffer */ + num_big_types = (int)(num_elements/bigio_count); + leftover = num_elements - num_big_types * (hsize_t)bigio_count; + H5_CHECKED_ASSIGN(remaining_bytes, int, leftover, hsize_t); + + /* Create a contiguous datatype of size equal to the largest + * number that a 32 bit integer can hold x size of old type. + * If the displacement is 0, then the type is contiguous, otherwise + * use type_hvector to create the type with the displacement provided + */ + if (0 == stride_bytes) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(bigio_count, + old_type, + &inner_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (bigio_count, + 1, + stride_bytes, + old_type, + &inner_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + + /* Create a contiguous datatype of the buffer (minus the remaining < 2GB part) + * If a stride is present, use hvector type + */ + if (0 == stride_bytes) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(num_big_types, + inner_type, + &outer_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (num_big_types, + 1, + stride_bytes, + inner_type, + &outer_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + + MPI_Type_free(&inner_type); + + /* If there is a remaining part create a contiguous/vector datatype and then + * use a struct datatype to encapsulate everything. + */ + if(remaining_bytes) { + if (stride_bytes == 0) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous (remaining_bytes, + old_type, + &leftover_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector + ((int)(num_elements - (hsize_t)num_big_types*bigio_count), + 1, + stride_bytes, + old_type, + &leftover_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + + MPI_Type_extent (old_type, &old_extent); + + /* Set up the arguments for MPI_Type_struct constructor */ + type[0] = outer_type; + type[1] = leftover_type; + block_len[0] = 1; + block_len[1] = 1; + disp[0] = 0; + disp[1] = (old_extent+stride_bytes)*num_big_types*(MPI_Aint)bigio_count; + + if(MPI_SUCCESS != (mpi_code = + MPI_Type_create_struct(2, block_len, disp, type, new_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + + MPI_Type_free(&outer_type); + MPI_Type_free(&leftover_type); + } + else { + /* There are no remaining bytes so just set the new type to + * the outer type created */ + *new_type = outer_type; + } + + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_mpio_create_large_type */ + #endif /* H5_HAVE_PARALLEL */ -- cgit v0.12 From 8935c921f7e50607cd91c86b2237ac39a9b600af Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Mon, 10 Jul 2017 03:22:48 -0500 Subject: Fix for HDFFV-10217 infinite loop in H5VM_power2up(). The function H5VM_power2up() returns the next power of 2 for n. When n exceeds 2^63, it overflows and becomes 0 causing the infinite looping. The fix ensures that the function checks for n >= 2^63 and returns 0. --- src/H5Dchunk.c | 7 +++- src/H5Ddeprec.c | 5 ++- src/H5Dint.c | 14 ++++++-- src/H5VMprivate.h | 6 +++- test/dsets.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 121 insertions(+), 8 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index d693466..b7b8b03 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1019,11 +1019,16 @@ H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, hid_t dapl_id) unsigned u; /* Local index value */ for(u = 0; u < dset->shared->ndims; u++) { + hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */ + /* Initial scaled dimension sizes */ rdcc->scaled_dims[u] = dset->shared->curr_dims[u] / dset->shared->layout.u.chunk.dim[u]; + if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Inital 'power2up' values for scaled dimensions */ - rdcc->scaled_power2up[u] = H5VM_power2up(rdcc->scaled_dims[u]); + rdcc->scaled_power2up[u] = scaled_power2up; /* Number of bits required to encode scaled dimension size */ rdcc->scaled_encode_bits[u] = H5VM_log2_gen(rdcc->scaled_power2up[u]); diff --git a/src/H5Ddeprec.c b/src/H5Ddeprec.c index 8d9461c..0807048 100644 --- a/src/H5Ddeprec.c +++ b/src/H5Ddeprec.c @@ -329,8 +329,11 @@ H5D__extend(H5D_t *dataset, const hsize_t *size, hid_t dxpl_id) dataset->shared->cache.chunk.scaled_dims[u] > dataset->shared->cache.chunk.nslots)) update_chunks = TRUE; + if( !(scaled_power2up = H5VM_power2up(scaled)) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Check if the number of bits required to encode the scaled size value changed */ - if(dataset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) { + if(dataset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) { /* Update the 'power2up' & 'encode_bits' values for the current dimension */ dataset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up; dataset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up); diff --git a/src/H5Dint.c b/src/H5Dint.c index 08b3eb8..3b938e2 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -745,8 +745,13 @@ H5D__cache_dataspace_info(const H5D_t *dset) dset->shared->ndims = (unsigned)sndims; /* Compute the inital 'power2up' values */ - for(u = 0; u < dset->shared->ndims; u++) - dset->shared->curr_power2up[u] = H5VM_power2up(dset->shared->curr_dims[u]); + for(u = 0; u < dset->shared->ndims; u++) { + hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */ + + if( !(scaled_power2up = H5VM_power2up(dset->shared->curr_dims[u])) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + dset->shared->curr_power2up[u] = scaled_power2up; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -2809,8 +2814,11 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id) dset->shared->cache.chunk.scaled_dims[u] > dset->shared->cache.chunk.nslots)) update_chunks = TRUE; + if( !(scaled_power2up = H5VM_power2up(scaled)) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Check if the number of bits required to encode the scaled size value changed */ - if(dset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) { + if(dset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) { /* Update the 'power2up' & 'encode_bits' values for the current dimension */ dset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up; dset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up); diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 4d71b29..decac7e 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -460,7 +460,11 @@ H5VM_power2up(hsize_t n) { hsize_t ret_value = 1; /* Return value */ - while(ret_value < n) + /* Returns 0 when n exceeds 2^63 */ + if(n >= (hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) - 1)) + ret_value = 0; + + while(ret_value && ret_value < n) ret_value <<= 1; return(ret_value); diff --git a/test/dsets.c b/test/dsets.c index c29d18e..0ca08e4 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -51,14 +51,15 @@ const char *FILENAME[] = { "copy_dcpl_newfile",/* 13 */ "partial_chunks", /* 14 */ "layout_extend", /* 15 */ - "zero_chunk", /* 16 */ + "zero_chunk", /* 16 */ "chunk_single", /* 17 */ "swmr_non_latest", /* 18 */ "earray_hdr_fd", /* 19 */ "farray_hdr_fd", /* 20 */ "bt2_hdr_fd", /* 21 */ - "storage_size", /* 22 */ + "storage_size", /* 22 */ "dls_01_strings", /* 23 */ + "power2up", /* 24 */ NULL }; #define FILENAME_BUF_SIZE 1024 @@ -11345,6 +11346,97 @@ error: /*------------------------------------------------------------------------- + * Function: test_power2up + * + * Purpose: Tests that the H5VM_power2up(n) function does not result in an + * infinite loop when input n exceeds 2^63. (HDFFV-10217) + * H5VM_power2up() is used to calculate the next power of 2 for + * a dataset's scaled dimension sizes. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Vailin Choi; June 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_power2up(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; + hid_t fid = -1; /* File ID */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + hsize_t dims[2]; /* Dataset dimension sizes */ + hsize_t max_dims[2]; /* Maximum dimension sizes */ + hsize_t chunk_dims[2]; /* Chunk dimensions */ + hsize_t ext_dims[2]; /* Extended dimension sizes */ + herr_t status; /* Error status */ + + TESTING("the next power of 2"); + + h5_fixname(FILENAME[24], fapl, filename, sizeof filename); + + /* Create file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR + + /* Set dims[1] to ((2^63) -1) */ + dims[0] = 0; + dims[1] = ((hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) -1)) - 1; + max_dims[0] = max_dims[1] = H5S_UNLIMITED; + sid = H5Screate_simple(2, dims, max_dims); + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + /* Set chunk size */ + chunk_dims[0] = chunk_dims[1] = 1; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + TEST_ERROR + + /* Create chunked dataset */ + if((did = H5Dcreate2(fid, "dset", H5T_NATIVE_INT64, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR + + ext_dims[0] = 1; + ext_dims[1] = dims[1] + 5; + + /* Extend to (2^63)+ */ + H5E_BEGIN_TRY { + status = H5Dset_extent(did, ext_dims); + } H5E_END_TRY; + if(status >= 0) + TEST_ERROR + + /* Closing */ + if(H5Dclose(did) < 0) + TEST_ERROR + if(H5Sclose(sid) < 0) + TEST_ERROR + if(H5Pclose(dcpl) < 0) + TEST_ERROR + if(H5Fclose(fid) < 0) + TEST_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} /* end test_power2up() */ + + +/*------------------------------------------------------------------------- * Function: test_scatter * * Purpose: Tests H5Dscatter with a variety of different selections @@ -12928,6 +13020,7 @@ main(void) nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); + nerrors += (test_power2up(my_fapl) < 0 ? 1 : 0); nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); -- cgit v0.12 From 0c4c562cc583fd814f26ba652cbcf82dc6d33cac Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Mon, 10 Jul 2017 16:17:26 -0400 Subject: Include code fixes and additional modifications pointed out by code reviewers --- MANIFEST | 1 + src/H5Smpio.c | 12 ++++----- testpar/CMakeLists.txt | 1 + testpar/Makefile.am | 2 +- testpar/t_bigio.c | 72 ++++++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 73 insertions(+), 15 deletions(-) diff --git a/MANIFEST b/MANIFEST index 475b674..27f38be 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1222,6 +1222,7 @@ ./testpar/COPYING ./testpar/Makefile.am +./testpar/t_bigio.c ./testpar/t_cache.c ./testpar/t_cache_image.c ./testpar/t_chunk_alloc.c diff --git a/src/H5Smpio.c b/src/H5Smpio.c index 7319a80..46f7a59 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -232,7 +232,7 @@ H5S_mpio_create_point_datatype (size_t elmt_size, hsize_t num_points, /* Create an MPI datatype for the whole point selection */ if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) #endif /* Commit MPI datatype for later use */ @@ -871,7 +871,7 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size, block_type, /* old type */ &outer_type); /* new type */ if(MPI_SUCCESS != mpi_code) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) } MPI_Type_free(&block_type); MPI_Type_free(&inner_type); @@ -1424,7 +1424,7 @@ static herr_t H5S_mpio_create_large_type (hsize_t num_elements, stride_bytes, old_type, &inner_type))) { - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) } } @@ -1444,7 +1444,7 @@ static herr_t H5S_mpio_create_large_type (hsize_t num_elements, stride_bytes, inner_type, &outer_type))) { - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) } } @@ -1468,7 +1468,7 @@ static herr_t H5S_mpio_create_large_type (hsize_t num_elements, stride_bytes, old_type, &leftover_type))) { - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) } } @@ -1484,7 +1484,7 @@ static herr_t H5S_mpio_create_large_type (hsize_t num_elements, if(MPI_SUCCESS != (mpi_code = MPI_Type_create_struct(2, block_len, disp, type, new_type))) { - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) } MPI_Type_free(&outer_type); diff --git a/testpar/CMakeLists.txt b/testpar/CMakeLists.txt index 298d326..e994b65 100644 --- a/testpar/CMakeLists.txt +++ b/testpar/CMakeLists.txt @@ -43,6 +43,7 @@ ENDMACRO (ADD_H5P_EXE file) set (H5P_TESTS t_mpi + t_bigio t_cache t_pflush1 t_pflush2 diff --git a/testpar/Makefile.am b/testpar/Makefile.am index b87c1df..7029bd5 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -23,7 +23,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame +TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame check_PROGRAMS = $(TEST_PROG_PARA) diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c index 2f80e45..830ea54 100644 --- a/testpar/t_bigio.c +++ b/testpar/t_bigio.c @@ -2060,13 +2060,62 @@ coll_chunktest(const char* filename, +/***************************************************************************** + * + * Function: do_express_test() + * + * Purpose: Do an MPI_Allreduce to obtain the maximum value returned + * by GetTestExpress() across all processes. Return this + * value. + * + * Envirmoment variables can be different across different + * processes. This function ensures that all processes agree + * on whether to do an express test. + * + * Return: Success: Maximum of the values returned by + * GetTestExpress() across all processes. + * + * Failure: -1 + * + * Programmer: JRM -- 4/25/06 + * + *****************************************************************************/ +static int +do_express_test(int world_mpi_rank) +{ + int express_test; + int max_express_test; + int result; + + express_test = GetTestExpress(); + + result = MPI_Allreduce((void *)&express_test, + (void *)&max_express_test, + 1, + MPI_INT, + MPI_MAX, + MPI_COMM_WORLD); + + if ( result != MPI_SUCCESS ) { + nerrors++; + max_express_test = -1; + if ( VERBOSE_MED && (world_mpi_rank == 0)) { + HDfprintf(stdout, "%d:%s: MPI_Allreduce() failed.\n", + world_mpi_rank, FUNC ); + } + } + + return(max_express_test); + +} /* do_express_test() */ int main(int argc, char **argv) { - + int ExpressMode = 0; hsize_t newsize = 1048576; hsize_t oldsize = H5S_mpio_set_bigio_count(newsize); + if (newsize != oldsize) { bigcount = newsize * 2; } @@ -2075,20 +2124,27 @@ int main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + ExpressMode = do_express_test(mpi_rank); + dataset_big_write(); MPI_Barrier(MPI_COMM_WORLD); dataset_big_read(); MPI_Barrier(MPI_COMM_WORLD); - oldsize = H5S_mpio_set_bigio_count(16384); + if (ExpressMode > 1) { + printf("***Express test mode on. Several tests are skipped\n"); + } + else { + coll_chunk1(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk2(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk3(); + } - coll_chunk1(); - MPI_Barrier(MPI_COMM_WORLD); - coll_chunk2(); - MPI_Barrier(MPI_COMM_WORLD); - coll_chunk3(); - MPI_Barrier(MPI_COMM_WORLD); + /* close HDF5 library */ + H5close(); MPI_Finalize(); -- cgit v0.12 From 32b0d6ca9f95fe46c2e52b58cabd4f5af4c107d3 Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Mon, 10 Jul 2017 18:20:39 -0400 Subject: Fix up the ExpressMode check for skipping slow running tests. --- testpar/t_bigio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c index 830ea54..a4a1323 100644 --- a/testpar/t_bigio.c +++ b/testpar/t_bigio.c @@ -2132,7 +2132,7 @@ int main(int argc, char **argv) dataset_big_read(); MPI_Barrier(MPI_COMM_WORLD); - if (ExpressMode > 1) { + if (ExpressMode > 0) { printf("***Express test mode on. Several tests are skipped\n"); } else { -- cgit v0.12 From 78d1de482ebc91348cfb673ba584db96ce127aeb Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 12 Jul 2017 14:18:01 -0500 Subject: HDFF-10254 - Copy test files with macro to avoid POST_BUILD failure --- hl/tools/h5watch/CMakeTests.cmake | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/hl/tools/h5watch/CMakeTests.cmake b/hl/tools/h5watch/CMakeTests.cmake index 35e7829..0b7b4d4 100644 --- a/hl/tools/h5watch/CMakeTests.cmake +++ b/hl/tools/h5watch/CMakeTests.cmake @@ -56,18 +56,11 @@ set (H5WATCH_TEST_FILES # make test dir file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") -add_custom_target(h5watch-files ALL COMMENT "Copying files needed by h5watch tests") foreach (h5watch_file ${H5WATCH_TEST_FILES}) - set (dest "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}") - #message (STATUS " Copying ${h5watch_file}") - add_custom_command ( - TARGET h5watch-files - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different ${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file} ${dest} - ) + HDFTEST_COPY_FILE("${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}" "H5WATCH_files") endforeach () +add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tests" DEPENDS ${H5WATCH_files_list}) ############################################################################## ############################################################################## -- cgit v0.12 From c22678f3fde05641ae00003272d9d1add600ba24 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 12 Jul 2017 14:23:00 -0500 Subject: HDFFV-10254 release note --- release_docs/RELEASE.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 20d58b3..c53f926 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -105,7 +105,15 @@ Bug Fixes since HDF5-1.10.1 release Configuration ------------- - - + - cmake + + To many commands for POST_BUILD step caused command line to be + too big on windows. + + Changed foreach of copy command to use a custom command with the + use of the HDFTEST_COPY_FILE macro. + + (ADB - 2017/07/12, HDFFV-10254) Performance ------------- @@ -127,7 +135,7 @@ Bug Fixes since HDF5-1.10.1 release The import from h5dump function expects the binary files to use native types (FILE '-b' option) in the binary file. - (ADB - 2017/06/15, HDFFV-102191) + (ADB - 2017/06/15, HDFFV-10219) - h5repack -- cgit v0.12 From 6a5aa46e936340ed540359290374fa909f9213a6 Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Thu, 13 Jul 2017 10:12:08 -0400 Subject: Added a brief outline for Large MPI-IO transfers into RELEASE.txt --- release_docs/RELEASE.txt | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 20d58b3..4335b37 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -62,7 +62,29 @@ New Features Parallel Library: ----------------- - - + - Large MPI-IO transfers + + Previous releases of PHDF5 would fail when attempting to + read or write greater than 2GB of data in a single IO operation. + This issue stems principally from an MPI API whose definitions + utilize 32 bit integers to describe the number of data elements + and datatype that MPI should use to effect a data transfer. + Historically, HDF5 has invoked MPI-IO with the number of + elements in a contiguous buffer represented as the length + of that buffer in bytes. + + Resolving the issue and thus enabling larger MPI-IO transfers + is accomplished first, by detecting when a user IO request would + exceed the 2GB limit as described above. Once a transfer request + is identified as requiring special handling, PHDF5 now creates a + derived datatype consisting of a vector of fixed sized blocks + which is in turn wrapped within a single MPI_Type_struct to + contain the vector and any remaining data. The newly created + datatype is then used in place of MPI_BYTE and can be used to + fulfill the original user request without encountering API + errors. + + (RAW – 2017/07/11, HDFFV-8839) Fortran Library: ---------------- -- cgit v0.12 From b3a212e54b86e15ea059d856db8685f43cc00829 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Thu, 13 Jul 2017 13:36:48 -0500 Subject: HDFFV-10254 Fix spelling --- release_docs/RELEASE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index c53f926..f83d3f4 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -107,7 +107,7 @@ Bug Fixes since HDF5-1.10.1 release ------------- - cmake - To many commands for POST_BUILD step caused command line to be + Too many commands for POST_BUILD step caused command line to be too big on windows. Changed foreach of copy command to use a custom command with the -- cgit v0.12 From d4234d0a98ff68eb55b3fa0d2246e61e53308e41 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 14 Jul 2017 09:18:33 -0700 Subject: Major rework of H5PL package code before bringing VOL changes over. Brings coding standards in line with the rest of the library, enforces better software engineering principles, and makes everything more maintainable. --- MANIFEST | 3 + bin/trace | 4 +- src/CMakeLists.txt | 3 + src/H5PL.c | 987 ++++++++++--------------------------------------- src/H5PLextern.h | 4 +- src/H5PLint.c | 381 +++++++++++++++++++ src/H5PLmodule.h | 10 +- src/H5PLpath.c | 776 ++++++++++++++++++++++++++++++++++++++ src/H5PLpkg.h | 113 +++++- src/H5PLplugin_cache.c | 308 +++++++++++++++ src/H5PLprivate.h | 4 +- src/H5PLpublic.h | 34 +- src/H5system.c | 50 +++ src/H5win32defs.h | 1 + src/Makefile.am | 2 +- test/plugin.c | 426 +++++++++++++-------- 16 files changed, 2131 insertions(+), 975 deletions(-) create mode 100644 src/H5PLint.c create mode 100644 src/H5PLpath.c create mode 100644 src/H5PLplugin_cache.c diff --git a/MANIFEST b/MANIFEST index 27f38be..739db4a 100644 --- a/MANIFEST +++ b/MANIFEST @@ -799,8 +799,11 @@ ./src/H5PBpkg.h ./src/H5PBprivate.h ./src/H5PL.c +./src/H5PLint.c ./src/H5PLmodule.h +./src/H5PLpath.c ./src/H5PLpkg.h +./src/H5PLplugin_cache.c ./src/H5PLprivate.h ./src/H5PLpublic.h ./src/H5PLextern.h diff --git a/bin/trace b/bin/trace index 3f532ab..cf41238 100755 --- a/bin/trace +++ b/bin/trace @@ -76,7 +76,7 @@ $Source = ""; "off_t" => "o", "H5O_type_t" => "Ot", "H5P_class_t" => "p", - "hobj_ref_t" => "r", + "hobj_ref_t" => "r", "H5R_type_t" => "Rt", "char" => "s", "unsigned char" => "s", @@ -124,7 +124,7 @@ $Source = ""; "H5G_iterate_t" => "x", "H5G_info_t" => "x", "H5I_free_t" => "x", - "H5I_search_func_t" => "x", + "H5I_search_func_t" => "x", "H5L_class_t" => "x", "H5L_elink_traverse_t" => "x", "H5L_iterate_t" => "x", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 178c954..96ea589 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c + ${HDF5_SRC_DIR}/H5PLint.c + ${HDF5_SRC_DIR}/H5PLpath.c + ${HDF5_SRC_DIR}/H5PLplugin_cache.c ) set (H5PL_HDRS diff --git a/src/H5PL.c b/src/H5PL.c index 65d6c91..fc42554 100644 --- a/src/H5PL.c +++ b/src/H5PL.c @@ -22,135 +22,28 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ -#include "H5MMprivate.h" /* Memory management */ #include "H5PLpkg.h" /* Plugin */ -#include "H5Zprivate.h" /* Filter pipeline */ /****************/ /* Local Macros */ /****************/ -#ifdef H5_HAVE_WIN32_API -#define H5PL_EXPAND_ENV_VAR { \ - long bufCharCount; \ - char *tempbuf; \ - if(NULL == (tempbuf = (char *)H5MM_malloc(H5PL_EXPAND_BUFFER_SIZE))) \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") \ - if((bufCharCount = ExpandEnvironmentStringsA(dl_path, tempbuf, H5PL_EXPAND_BUFFER_SIZE)) > H5PL_EXPAND_BUFFER_SIZE) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") \ - } \ - if(bufCharCount == 0) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") \ - } \ - dl_path = (char *)H5MM_xfree(dl_path); \ - dl_path = tempbuf; \ - } -#else -#define H5PL_EXPAND_ENV_VAR -#endif /* H5_HAVE_WIN32_API */ - -/****************************/ -/* Macros for supporting - * both Windows and Unix */ -/****************************/ -/* Windows support - * - * SPECIAL WINDOWS NOTE - * - * Some of the Win32 API functions expand to fooA or fooW depending on - * whether UNICODE or _UNICODE are defined. You MUST explicitly use - * the A version of the functions to force char * behavior until we - * work out a scheme for proper Windows Unicode support. - * - * If you do not do this, people will be unable to incorporate our - * source code into their own CMake builds if they define UNICODE. - */ -#ifdef H5_HAVE_WIN32_API - -#define H5PL_PATH_SEPARATOR ";" - -/* Handle for dynamic library */ -#define H5PL_HANDLE HINSTANCE - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) FreeLibrary(H) - -/* Clear error - nothing to do */ -#define H5PL_CLR_ERROR - -/* maximum size for expanding env vars */ -#define H5PL_EXPAND_BUFFER_SIZE 32767 - -typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); - -/* Unix support */ -#else /* H5_HAVE_WIN32_API */ - -#define H5PL_PATH_SEPARATOR ":" - -/* Handle for dynamic library */ -#define H5PL_HANDLE void * - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) dlclose(H) - -/* Clear error */ -#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) - -typedef const void *(*H5PL_get_plugin_info_t)(void); -#endif /* H5_HAVE_WIN32_API */ - -/* Whether to preload pathnames for plugin libraries */ -#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR - -/* Special symbol to indicate no plugin loading */ -#define H5PL_NO_PLUGIN "::" /******************/ /* Local Typedefs */ /******************/ -/* Type for the list of info for opened plugin libraries */ -typedef struct H5PL_table_t { - H5PL_type_t pl_type; /* plugin type */ - int pl_id; /* ID for the plugin */ - H5PL_HANDLE handle; /* plugin handle */ -} H5PL_table_t; - /********************/ /* Local Prototypes */ /********************/ -static herr_t H5PL__init_path_table(void); -static htri_t H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info); -static htri_t H5PL__open(H5PL_type_t pl_type, char *libname, int plugin_id, const void **pl_info); -static htri_t H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info); -static herr_t H5PL__close(H5PL_HANDLE handle); - /*********************/ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -161,145 +54,43 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Local Variables */ /*******************/ -/* Table for opened plugin libraries */ -static size_t H5PL_table_alloc_g = 0; -static size_t H5PL_table_used_g = 0; -static H5PL_table_t *H5PL_table_g = NULL; - -/* Table of location paths for plugin libraries */ -static char *H5PL_path_table_g[H5PL_MAX_PATH_NUM]; -static size_t H5PL_num_paths_g = 0; -static hbool_t H5PL_path_found_g = FALSE; - -/* Enable all plugin libraries */ -static unsigned int H5PL_plugin_g = H5PL_ALL_PLUGIN; - - - -/*-------------------------------------------------------------------------- -NAME - H5PL__init_package -- Initialize interface-specific information -USAGE - herr_t H5PL__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5PL__init_package(void) -{ - char *preload_path; - - FUNC_ENTER_PACKAGE_NOERR - - /* Retrieve pathnames from HDF5_PLUGIN_PRELOAD if the user sets it - * to tell the library to load plugin libraries without search. - */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__init_package() */ /*------------------------------------------------------------------------- - * Function: H5PL_term_package - * - * Purpose: Terminate the H5PL interface: release all memory, reset all - * global variables to initial values. This only happens if all - * types have been destroyed from other interfaces. + * Function: H5PLset_loading_state * - * Return: Success: Positive if any action was taken that might - * affect some other interface; zero otherwise. - * Failure: Negative. - * - * Programmer: Raymond Lu - * 20 February 2013 - * - *------------------------------------------------------------------------- - */ -int -H5PL_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - size_t u; /* Local index variable */ - - /* Close opened dynamic libraries */ - if(H5PL_table_g) { - for(u = 0; u < H5PL_table_used_g; u++) - H5PL__close((H5PL_table_g[u]).handle); - - /* Free the table of dynamic libraries */ - H5PL_table_g = (H5PL_table_t *)H5MM_xfree(H5PL_table_g); - H5PL_table_used_g = H5PL_table_alloc_g = 0; - - n++; - } /* end if */ - - /* Free the table of search paths */ - if(H5PL_num_paths_g > 0) { - for(u = 0; u < H5PL_num_paths_g; u++) - if(H5PL_path_table_g[u]) - H5PL_path_table_g[u] = (char *)H5MM_xfree(H5PL_path_table_g[u]); - H5PL_num_paths_g = 0; - H5PL_path_found_g = FALSE; - - n++; - } /* end if */ - - /* Mark the interface as uninitialized */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5PL_term_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLset_loading_state + * Purpose: Control the loading of dynamic plugin types. * - * Purpose: Control the loading of dynamic plugin types. + * The plugin_control_mask parameter is a bitfield that controls + * whether certain classes of plugins (e.g.: filters, + * VOL drivers) will be loaded by the library. * - * This function will not allow plugin types if the pathname from the HDF5_PLUGIN_PRELOAD - * environment variable is set to the special "::" string. + * plugin bit = 0, will prevent the use of that dynamic plugin type. + * plugin bit = 1, will allow the use of that dynamic plugin type. * - * plugin bit = 0, will prevent the use of that dynamic plugin type. - * plugin bit = 1, will allow the use of that dynamic plugin type. + * A list of pre-defined masks can be found in H5PLpublic.h. + * Set the mask to 0 to disable all plugins. * - * H5PL_TYPE_FILTER changes just dynamic filters - * A H5PL_ALL_PLUGIN will enable all dynamic plugin types - * A zero value will disable all dynamic plugin types + * This function will not allow plugin types if the pathname + * from the HDF5_PLUGIN_PRELOAD environment variable is set to + * the special "::" string. * - * Return: Non-negative or success + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLset_loading_state(unsigned int plugin_type) +H5PLset_loading_state(unsigned int plugin_control_mask) { - char *preload_path; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "Iu", plugin_type); - - /* change the bit value of the requested plugin type(s) */ - H5PL_plugin_g = plugin_type; + H5TRACE1("e", "Iu", plugin_control_mask); - /* check if special ENV variable is set and disable all plugin types */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; + /* Set the plugin control mask */ + if(H5PL__set_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTSET, FAIL, "error setting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -307,27 +98,35 @@ done: /*------------------------------------------------------------------------- - * Function: H5PLget_loading_state + * Function: H5PLget_loading_state * - * Purpose: Query state of the loading of dynamic plugin types. + * Purpose: Get the bitmask that controls whether certain classes + * of plugins (e.g.: filters, VOL drivers) will be loaded + * by the library. * - * This function will return the state of the global flag. + * Zero if all plugin types are disabled + * Negative if all plugin types are enabled + * Positive if one or more of the plugin types are enabled * - * Return: Zero if all plugin types are disabled, negative if all - * plugin types are enabled, positive if one or more of the plugin types are enabled. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLget_loading_state(unsigned int *plugin_type) +H5PLget_loading_state(unsigned int *plugin_control_mask) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", plugin_type); + H5TRACE1("e", "*Iu", plugin_control_mask); + + if (NULL == plugin_control_mask) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_control_mask parameter cannot be NULL") - if(plugin_type) - *plugin_type = H5PL_plugin_g; + /* Set the plugin control mask */ + if(H5PL__get_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error getting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -335,674 +134,282 @@ done: /*------------------------------------------------------------------------- - * Function: H5PL_load - * - * Purpose: Given the plugin type and identifier, this function searches - * and/or loads a dynamic plugin library first among the already - * opened libraries then in the designated location paths. - * - * Return: Non-NULL on success/NULL on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -const void * -H5PL_load(H5PL_type_t type, int id) -{ - htri_t found; /* Whether the plugin was found */ - const void *plugin_info = NULL; - const void *ret_value = NULL; - - FUNC_ENTER_NOAPI(NULL) - - switch(type) { - case H5PL_TYPE_FILTER: - if((H5PL_plugin_g & H5PL_FILTER_PLUGIN) == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) - break; - - case H5PL_TYPE_ERROR: - case H5PL_TYPE_NONE: - default: - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) - } /* end switch */ - - /* Initialize the location paths for dynamic libraries, if they aren't - * already set up. - */ - if(FALSE == H5PL_path_found_g) - if(H5PL__init_path_table() < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, NULL, "can't initialize search path table") - - /* Search in the table of already loaded plugin libraries */ - if((found = H5PL__search_table(type, id, &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in table failed") - - /* If not found, iterate through the path table to find the right dynamic library */ - if(!found) { - size_t i; /* Local index variable */ - - for(i = 0; i < H5PL_num_paths_g; i++) { - if((found = H5PL__find(type, id, H5PL_path_table_g[i], &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in paths failed") - - /* Break out if found */ - if(found) { - HDassert(plugin_info); - break; - } /* end if */ - } /* end for */ - } /* end if */ - - /* Check if we found the plugin */ - if(found) - ret_value = plugin_info; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL_load() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLappend + * Function: H5PLappend * - * Purpose: Insert a plugin path at the end of the list. + * Purpose: Insert a plugin search path at the end of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLappend(const char *plugin_path) +H5PLappend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - H5PL_path_table_g[H5PL_num_paths_g] = dl_path; - H5PL_num_paths_g++; + /* Append the search path to the path table */ + if (H5PL__append_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTAPPEND, FAIL, "unable to append search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLappend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLprepend + * Function: H5PLprepend * - * Purpose: Insert a plugin path at the beginning of the list. + * Purpose: Insert a plugin search path at the beginning of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLprepend(const char *plugin_path) +H5PLprepend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - for (plindex = (unsigned int)H5PL_num_paths_g; plindex > 0; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[0] = dl_path; - H5PL_num_paths_g++; + /* Prepend the search path to the path table */ + if (H5PL__prepend_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLprepend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLreplace + * Function: H5PLreplace * - * Purpose: Replace the path at the specified index. + * Purpose: Replace the path at the specified index. The path at the + * index must exist. * - * Return: Non-negative or success. + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t -H5PLreplace(const char *plugin_path, unsigned int index) +H5PLreplace(const char *search_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "*sIu", plugin_path, index); - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE2("e", "*sIu", search_path, index); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - if(H5PL_path_table_g[index]) - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - H5PL_path_table_g[index] = dl_path; + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__replace_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLreplace() */ - + /*------------------------------------------------------------------------- - * Function: H5PLinsert + * Function: H5PLinsert * - * Purpose: Insert a plugin path at the specified index, moving other paths after the index. + * Purpose: Insert a plugin search path at the specified index, moving + * other paths after the index. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLinsert(const char *plugin_path, unsigned int index) +H5PLinsert(const char *search_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "*sIu", plugin_path, index); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - for(plindex = (unsigned int)H5PL_num_paths_g; plindex > index; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[index] = dl_path; - H5PL_num_paths_g++; + H5TRACE2("e", "*sIu", search_path, index); + + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if ((0 != num_paths) && (index >= num_paths)) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__insert_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLinsert() */ - + /*------------------------------------------------------------------------- - * Function: H5PLremove + * Function: H5PLremove + * + * Purpose: Remove the plugin path at the specifed index and compact + * the list. * - * Purpose: Remove the plugin path at the specifed index and compacting the list. + * Return: Success: Non-negative + * Failture: Negative * - * Return: Non-negative or success. + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t H5PLremove(unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "Iu", index); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == H5PL_path_table_g[index]) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - - H5PL_num_paths_g--; - for(plindex = index; plindex < (unsigned int)H5PL_num_paths_g; plindex++) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex + 1]; - H5PL_path_table_g[H5PL_num_paths_g] = NULL; + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Delete the search path from the path table */ + if (H5PL__remove_path(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "unable to remove search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLremove() */ - + /*------------------------------------------------------------------------- - * Function: H5PLget + * Function: H5PLget + * + * Purpose: Query the plugin path at a specified index. + * + * If 'path_buf' is non-NULL then up to 'buf_size' bytes will be written into + * that buffer and the length of the path name will be returned. * - * Purpose: Query the plugin path at the specified index. + * If 'path_buf' is NULL, this function will simply return the number of + * characters required to store the path name, ignoring 'path_buf' and + * 'buf_size' * - * Return: Success: The length of path. + * If an error occurs then the buffer pointed to by 'path_buf' + * (NULL or non-NULL) will be unchanged and the function will return a + * negative value. * - * If `pathname' is non-NULL then write up to `size' bytes into that - * buffer and always return the length of the pathname. - * Otherwise `size' is ignored and the function does not store the pathname, - * just returning the number of characters required to store the pathname. - * If an error occurs then the buffer pointed to by `pathname' (NULL or non-NULL) - * is unchanged and the function returns a negative value. - * If a zero is returned for the name's length, then there is no pathname - * associated with the index. + * If a zero is returned for the name's length, then there is no path name + * associated with the index and the 'path_buf' buffer will be unchanged. + * + * Return: Success: The length of path + * Failure: A negative value * *------------------------------------------------------------------------- */ ssize_t -H5PLget(unsigned int index, char *pathname/*out*/, size_t size) +H5PLget(unsigned int index, char *path_buf, size_t buf_size) { - ssize_t ret_value = 0; /* Return value */ - size_t len = 0; /* Length of pathname */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + const char *path = NULL; /* path from table */ + size_t path_len = 0; /* Length of path */ + ssize_t ret_value = 0; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("Zs", "Iuxz", index, pathname, size); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5PL_path_table_g[index])) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - len = HDstrlen(dl_path); - if(pathname) { - HDstrncpy(pathname, dl_path, MIN((size_t)(len + 1), size)); - if((size_t)len >= size) - pathname[size - 1] = '\0'; + H5TRACE3("Zs", "Iu*sz", index, path_buf, buf_size); + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Check if the search table is empty */ + if (H5PL__get_num_paths() == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, (-1), "plugin search path table is empty") + + /* Get the path at the specified index and its length */ + if (NULL == (path = H5PL__get_path(index))) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, (-1), "no path stored at that index") + path_len = HDstrlen(path); + + /* If the path buffer is not NULL, copy the path to the buffer */ + if (path_buf) { + HDstrncpy(path_buf, path, MIN((size_t)(path_len + 1), buf_size)); + if ((size_t)path_len >= buf_size) + path_buf[buf_size - 1] = '\0'; } /* end if */ /* Set return value */ - ret_value = (ssize_t)len; + ret_value = (ssize_t)path_len; done: FUNC_LEAVE_API(ret_value) } /* end H5PLget() */ - + /*------------------------------------------------------------------------- - * Function: H5PLsize + * Function: H5PLsize * - * Purpose: Query the size of the current list of plugin paths. + * Purpose: Get the number of stored plugin paths. + * XXX: This is a terrible name. Can it be changed? * - * Return: Plugin path size + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ herr_t -H5PLsize(unsigned int *listsize) +H5PLsize(unsigned int *num_paths) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", listsize); + H5TRACE1("e", "*Iu", num_paths); - *listsize = (unsigned int)H5PL_num_paths_g; + /* Check arguments */ + if (!num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "num_paths parameter cannot be NULL") + + /* Get the number of stored plugin paths */ + *num_paths = H5PL__get_num_paths(); done: FUNC_LEAVE_API(ret_value) } /* end H5PLsize() */ - -/*------------------------------------------------------------------------- - * Function: H5PL__init_path_table - * - * Purpose: Initialize the path table. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * 18 March 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__init_path_table(void) -{ - char *dl_path = NULL; - char *origin_dl_path; - char *dir; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it - * or from the default paths if it isn't set. - */ - origin_dl_path = HDgetenv("HDF5_PLUGIN_PATH"); - if(NULL == origin_dl_path) - dl_path = H5MM_strdup(H5PL_DEFAULT_PATH); - else - dl_path = H5MM_strdup(origin_dl_path); - if(NULL == dl_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - /* Put paths in the path table. They are separated by ":" */ - dir = HDstrtok(dl_path, H5PL_PATH_SEPARATOR); - while(dir) { - /* Check for too many directories in path */ - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == (H5PL_path_table_g[H5PL_num_paths_g] = H5MM_strdup(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - H5PL_num_paths_g++; - dir = HDstrtok(NULL, H5PL_PATH_SEPARATOR); - } /* end while */ - - H5PL_path_found_g = TRUE; - -done: - if(dl_path) - dl_path = (char *)H5MM_xfree(dl_path); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__init_path_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__find - * - * Purpose: Given a path, this function opens the directory and envokes - * another function to go through all files to find the right - * plugin library. Two function definitions are for Unix and - * Windows. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -#ifndef H5_HAVE_WIN32_API -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - char *pathname = NULL; - DIR *dirp = NULL; - struct dirent *dp; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Open the directory */ - if(!(dirp = HDopendir(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) - - /* Iterates through all entries in the directory to find the right plugin library */ - while(NULL != (dp = HDreaddir(dirp))) { - /* The library we are looking for should be called libxxx.so... on Unix - * or libxxx.xxx.dylib on Mac. - */ -#ifndef __CYGWIN__ - if(!HDstrncmp(dp->d_name, "lib", (size_t)3) && - (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { -#else - if(!HDstrncmp(dp->d_name, "cyg", (size_t)3) && - HDstrstr(dp->d_name, ".dll") ) { - -#endif - h5_stat_t my_stat; - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(dp->d_name) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s/%s", dir, dp->d_name); - - /* Get info for directory entry */ - if(HDstat(pathname, &my_stat) == -1) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file: %s", HDstrerror(errno)) - - /* If it is a directory, skip it */ - if(S_ISDIR(my_stat.st_mode)) - continue; - - /* Attempt to open the dynamic library as a filter library */ - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } /* end while */ - -done: - if(dirp) - if(HDclosedir(dirp) < 0) - HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#else /* H5_HAVE_WIN32_API */ -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - WIN32_FIND_DATAA fdFile; - HANDLE hFind; - char *pathname = NULL; - char service[2048]; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Specify a file mask. *.* = We want everything! */ - sprintf(service, "%s\\*.dll", dir); - if((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") - - do { - /* Find first file will always return "." - * and ".." as the first two directories. - */ - if(HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(fdFile.cFileName) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s\\%s", dir, fdFile.cFileName); - - /* Is the entity a File or Folder? */ - if(fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - continue; - - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } while(FindNextFileA(hFind, &fdFile)); /* Find the next file. */ - -done: - if(hFind) - FindClose(hFind); - if(pathname) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#endif /* H5_HAVE_WIN32_API */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__open - * - * Purpose: Iterates through all files to find the right plugin library. - * It loads the dynamic plugin library and keeps it on the list - * of loaded libraries. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__open(H5PL_type_t pl_type, char *libname, int pl_id, const void **pl_info) -{ - H5PL_HANDLE handle = NULL; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* There are different reasons why a library can't be open, e.g. wrong architecture. - * simply continue if we can't open it. - */ - if(NULL == (handle = H5PL_OPEN_DLIB(libname))) { - H5PL_CLR_ERROR; /* clear error */ - } /* end if */ - else { - H5PL_get_plugin_info_t get_plugin_info = NULL; - - /* Return a handle for the function H5PLget_plugin_info in the dynamic library. - * The plugin library is suppose to define this function. - */ - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - else { - const H5Z_class2_t *plugin_info; - - /* Invoke H5PLget_plugin_info to verify this is the right library we are looking for. - * Move on if it isn't. - */ - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") - } /* end if */ - - /* Successfully found plugin library, check if it's the right one */ - if(plugin_info->id == pl_id) { - /* Expand the table if it is too small */ - if(H5PL_table_used_g >= H5PL_table_alloc_g) { - size_t n = MAX(H5Z_MAX_NFILTERS, 2 * H5PL_table_alloc_g); - H5PL_table_t *table = (H5PL_table_t *)H5MM_realloc(H5PL_table_g, n * sizeof(H5PL_table_t)); - - if(!table) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend dynamic library table") - - H5PL_table_g = table; - H5PL_table_alloc_g = n; - } /* end if */ - - (H5PL_table_g[H5PL_table_used_g]).handle = handle; - (H5PL_table_g[H5PL_table_used_g]).pl_type = pl_type; - (H5PL_table_g[H5PL_table_used_g]).pl_id = plugin_info->id; - H5PL_table_used_g++; - - /* Set the plugin info to return */ - *pl_info = (const void *)plugin_info; - - /* Indicate success */ - ret_value = TRUE; - } /* end if */ - else - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__open() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__search_table - * - * Purpose: Search in the list of already opened dynamic libraries - * to see if the one we are looking for is already opened. - * - * Return: TRUE on success, - * FALSE on not found, - * Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info) -{ - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Search in the table of already opened dynamic libraries */ - if(H5PL_table_used_g > 0) { - size_t i; - - for(i = 0; i < H5PL_table_used_g; i++) { - if((plugin_type == (H5PL_table_g[i]).pl_type) && (type_id == (H5PL_table_g[i]).pl_id)) { - H5PL_get_plugin_info_t get_plugin_info; - const H5Z_class2_t *plugin_info; - - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_table_g[i]).handle, "H5PLget_plugin_info"))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") - - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") - - *info = plugin_info; - HGOTO_DONE(TRUE) - } /* end if */ - } /* end for */ - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__search_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__close - * - * Purpose: Closes the handle for dynamic library - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__close(H5PL_HANDLE handle) -{ - FUNC_ENTER_STATIC_NOERR - - H5PL_CLOSE_LIB(handle); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__close() */ - diff --git a/src/H5PLextern.h b/src/H5PLextern.h index 7547ad7..cd5464d 100644 --- a/src/H5PLextern.h +++ b/src/H5PLextern.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu - * 13 February 2013 + * Purpose: Header file for writing external HDF5 plugins. */ + #ifndef _H5PLextern_H #define _H5PLextern_H diff --git a/src/H5PLint.c b/src/H5PLint.c new file mode 100644 index 0000000..bd6bf7d --- /dev/null +++ b/src/H5PLint.c @@ -0,0 +1,381 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Internal routines for managing plugins. + * + */ + + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Bitmask that controls whether classes of plugins + * (e.g.: filters, VOL drivers) can be loaded. + */ +static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN; + +/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD + * environment variable was set to H5PL_NO_PLUGIN at + * package initialization. + */ +static hbool_t H5PL_allow_plugins_g = TRUE; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_plugin_control_mask + * + * Purpose: Gets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__get_plugin_control_mask(unsigned int *mask /*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args - Just assert on package functions */ + HDassert(mask); + + /* Return the mask */ + *mask = H5PL_plugin_control_mask_g; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__get_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__set_plugin_control_mask + * + * Purpose: Sets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__set_plugin_control_mask(unsigned int mask) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Only allow setting this if plugins have not been disabled. + * XXX: Note that we don't consider this an error, but instead + * silently ignore it. We may want to consider this behavior + * more carefully. + */ + if (H5PL_allow_plugins_g) + H5PL_plugin_control_mask_g = mask; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__set_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__init_package + * + * Purpose: Initialize any package-specific data and call any init + * routines for the package. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__init_package(void) +{ + char *env_var = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the environment variable to determine if the user wants + * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in + * H5PLpublic.h) means we don't want to load plugins. + */ + if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD"))) + if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) { + H5PL_plugin_control_mask_g = 0; + H5PL_allow_plugins_g = FALSE; + } + + /* Create the table of previously-loaded plugins */ + if (H5PL__create_plugin_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache") + + /* Create the table of search paths for dynamic libraries */ + if (H5PL__create_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_term_package + * + * Purpose: Terminate the H5PL interface: release all memory, reset all + * global variables to initial values. This only happens if all + * types have been destroyed from other interfaces. + * + * Return: Success: Positive if any action was taken that might + * affect some other interface; zero otherwise + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5PL_term_package(void) +{ + hbool_t already_closed = FALSE; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + + if (H5_PKG_INIT_VAR) { + + /* Close the plugin cache. + * We need to bump the return value if we did any real work here. + */ + if (H5PL__close_plugin_cache(&already_closed) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache") + if (!already_closed) + ret_value++; + + /* Close the search path table and free the paths */ + if (H5PL__close_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table") + + /* Mark the interface as uninitialized */ + if (0 == ret_value) + H5_PKG_INIT_VAR = FALSE; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_load + * + * Purpose: Given the plugin type and identifier, this function searches + * for and, if found, loads a dynamic plugin library. + * + * The function searches first in the cached plugins and then + * in the paths listed in the path table. + * + * Return: Success: A pointer to the plugin info + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const void * +H5PL_load(H5PL_type_t type, int id) +{ + H5PL_search_params_t search_params; + hbool_t found = FALSE; /* Whether the plugin was found */ + const void *plugin_info = NULL; + const void *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + /* Check if plugins can be loaded for this plugin type */ + switch (type) { + case H5PL_TYPE_FILTER: + if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) + break; + + case H5PL_TYPE_ERROR: + case H5PL_TYPE_NONE: + default: + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) + } + + /* Set up the search parameters */ + search_params.type = type; + search_params.id = id; + + /* Search in the table of already loaded plugin libraries */ + if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed") + + /* If not found, try iterating through the path table to find an appropriate plugin */ + if (!found) + if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed") + + /* Set the return value we found the plugin */ + if (found) + ret_value = plugin_info; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_load() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__open + * + * Purpose: Opens a plugin. + * + * The success parameter will be set to TRUE and the plugin_info + * parameter will be filled in on success. Otherwise, they + * will be FALSE and NULL, respectively. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info) +{ + H5PL_HANDLE handle = NULL; + H5PL_get_plugin_info_t get_plugin_info = NULL; + htri_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(success); + HDassert(plugin_info); + + /* Initialize out parameters */ + *success = FALSE; + *plugin_info = NULL; + + /* There are different reasons why a library can't be open, e.g. wrong architecture. + * If we can't open the library, just return. + */ + if (NULL == (handle = H5PL_OPEN_DLIB(path))) { + H5PL_CLR_ERROR; /* clear error */ + HGOTO_DONE(SUCCEED); + } + + + /* Return a handle for the function H5PLget_plugin_info in the dynamic library. + * The plugin library is suppose to define this function. + * + * NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting + * object pointers to function pointers, which is undefined in ANSI C. + * This is basically unavoidable due to the nature of dlsym() and *is* + * defined in POSIX, so it's fine. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { +#pragma GCC diagnostic pop + + const H5Z_class2_t *info; + + /* Get the plugin info */ + if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)())) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Check if the filter IDs match */ + if (info->id == id) { + + /* Store the plugin in the cache */ + if (H5PL__add_plugin(type, id, handle)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache") + + /* Set output parameters */ + *success = TRUE; + *plugin_info = (const void *)info; + } + } + +done: + if (!success && handle) + if (H5PL__close(handle) < 0) + HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__open() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close + * + * Purpose: Closes the handle for dynamic library + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close(H5PL_HANDLE handle) +{ + FUNC_ENTER_PACKAGE_NOERR + + H5PL_CLOSE_LIB(handle); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5PL__close() */ + diff --git a/src/H5PLmodule.h b/src/H5PLmodule.h index b441aed..945441e 100644 --- a/src/H5PLmodule.h +++ b/src/H5PLmodule.h @@ -11,13 +11,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol - * Saturday, September 12, 2015 - * - * Purpose: This file contains declarations which define macros for the - * H5PL package. Including this header means that the source file - * is part of the H5PL package. + * Purpose: This file contains declarations which define macros for the + * H5PL package. Including this header means that the source file + * is part of the H5PL package. */ + #ifndef _H5PLmodule_H #define _H5PLmodule_H diff --git a/src/H5PLpath.c b/src/H5PLpath.c new file mode 100644 index 0000000..435802a --- /dev/null +++ b/src/H5PLpath.c @@ -0,0 +1,776 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a path table which stores plugin search paths. + * + * The path table is implemented as a dynamic, global array which + * will grow as new paths are inserted. The capacity of the path + * table never shrinks (though given the low number of paths + * expected and the low likelihood of paths being removed, this + * seems unlikely to be a problem). Inserts and removals rework + * the array so that there are no 'holes' in the in-use part + * of the array. + * + * Note that it's basically up to the user to manage the indexes + * when a complicated series of insert, overwrite, and, remove + * operations take place. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the path table */ +#define H5PL_INITIAL_PATH_CAPACITY 16 + +/* The amount to add to the capacity when the table is full */ +#define H5PL_PATH_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__insert_at(const char *path, unsigned int index); +static herr_t H5PL__make_space_at(unsigned int index); +static herr_t H5PL__replace_at(const char *path, unsigned int index); +static herr_t H5PL__expand_path_table(void); +static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info); + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Stored plugin paths to search */ +static char **H5PL_paths_g = NULL; + +/* The number of stored paths */ +static unsigned H5PL_num_paths_g = 0; + +/* The capacity of the path table */ +static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_at() + * + * Purpose: Insert a path at a particular index in the path table. + * Does not clobber! Will move existing paths up to make + * room. Use H5PL__replace_at(index) if you want to clobber. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__insert_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Expand the table if it is full */ + if (H5PL_num_paths_g == H5PL_path_capacity_g) + if (H5PL__expand_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table") + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if(H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* If the table entry is in use, make some space */ + if (H5PL_paths_g[index]) + if (H5PL__make_space_at(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry") + + /* Insert the copy of the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + H5PL_num_paths_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__make_space_at() + * + * Purpose: Free up a slot in the path table, moving existing path + * entries as necessary. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__make_space_at(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Copy the paths back to make a space */ + for (u = H5PL_num_paths_g; u > index; u--) + H5PL_paths_g[u] = H5PL_paths_g[u-1]; + + H5PL_paths_g[index] = NULL; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__make_space_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_at() + * + * Purpose: Replace a path at a particular index in the path table. + * The path in the table must exist and will be freed by this + * function. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__replace_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Check that the table entry is in use */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index) + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if (H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* Free the existing path entry */ + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Copy the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_path_table + * + * Purpose: Create the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_path_table(void) +{ + char *env_var= NULL; /* Path string from environment variable */ + char *paths = NULL; /* Delimited paths string. Either from the + * environment variable or the default. + */ + char *next_path = NULL; /* A path tokenized from the paths string */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the path table */ + H5PL_num_paths_g = 0; + H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table") + + /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it + * or from the default paths if it isn't set. + */ + env_var = HDgetenv("HDF5_PLUGIN_PATH"); + if (NULL == env_var) + paths = H5MM_strdup(H5PL_DEFAULT_PATH); + else + paths = H5MM_strdup(env_var); + + if (NULL == paths) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy") + + /* Separate the paths and store them */ + /* XXX: strtok() is not thread-safe */ + next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR); + while (next_path) { + + /* Insert the path into the table */ + if (H5PL__append_path(next_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path) + + /* Get the next path from the environment string */ + next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR); + } /* end while */ + +done: + if (paths) + paths = (char *)H5MM_xfree(paths); + + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_paths_g) + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + H5PL_path_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_path_table + * + * Purpose: Close the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_path_table(void) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Free paths */ + for (u = 0; u < H5PL_num_paths_g; u++) + if (H5PL_paths_g[u]) + H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]); + + /* Free path table */ + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + + /* Reset values */ + H5PL_num_paths_g = 0; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__close_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_num_paths + * + * Purpose: Gets the number of plugin paths that have been stored. + * + * Return: Success: The number of paths + * Failture: Can't fail + *------------------------------------------------------------------------- + */ +unsigned +H5PL__get_num_paths(void) +{ + FUNC_ENTER_PACKAGE_NOERR + + FUNC_LEAVE_NOAPI(H5PL_num_paths_g) + +} /* end H5PL__get_num_paths() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_path_table + * + * Purpose: Expand the path table when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_path_table(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed") + + /* Initialize the new memory */ + HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *)); + +done: + /* Set the path capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__append_path + * + * Purpose: Insert a path at the end of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__append_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the end of the table */ + if (H5PL__insert_at(path, H5PL_num_paths_g) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__append_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__prepend_path + * + * Purpose: Insert a path at the beginning of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__prepend_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the beginning of the table */ + if (H5PL__insert_at(path, 0) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__prepend_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_path + * + * Purpose: Replace a path at particular index in the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__replace_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__replace_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_path + * + * Purpose: Insert a path at particular index in the table, moving + * any existing paths back to make space. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__insert_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__insert_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__remove_path + * + * Purpose: Remove a path at particular index in the table, freeing + * the path string and moving the paths down to close the gap. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__remove_path(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Check if the path at that index is set */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index) + + /* Delete the path */ + H5PL_num_paths_g--; + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Shift the paths down to close the gap */ + for (u = index; u < H5PL_num_paths_g; u++) + H5PL_paths_g[u] = H5PL_paths_g[u+1]; + + /* Set the (former) last path to NULL */ + H5PL_paths_g[H5PL_num_paths_g] = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__remove_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_path + * + * Purpose: Get a pointer to a path at particular index in the table. + * + * Return: Success: A pointer to a path string stored in the table + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const char * +H5PL__get_path(unsigned int index) +{ + char *ret_value = NULL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Get the path at the requested index */ + if (index >= H5PL_num_paths_g) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index) + + return H5PL_paths_g[index]; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path_table + * + * Purpose: Attempts to find a matching plugin in the file system + * using the paths stored in the path table. + *. + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over the paths in the table, checking for an appropriate plugin */ + for (u = 0; u < H5PL_num_paths_g; u++) { + + /* Search for the plugin in this path */ + if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u]) + + /* Break out if found */ + if (*found) { + if (!plugin_info) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL") + break; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path + * + * Purpose: Given a path, this function opens the directory and envokes + * another function to go through all files to find the right + * plugin library. Two function definitions are for Unix and + * Windows. + * + * The found parameter will be set to TRUE and the info + * parameter will be filled in on success. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +#ifndef H5_HAVE_WIN32_API +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + char *path = NULL; + DIR *dirp = NULL; /* Directory stream */ + struct dirent *dp = NULL; /* Directory entry */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Open the directory */ + if (!(dirp = HDopendir(dir))) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) + + /* Iterate through all entries in the directory */ + while (NULL != (dp = HDreaddir(dirp))) { + + /* The library we are looking for should be called libxxx.so... on Unix + * or libxxx.xxx.dylib on Mac. + */ +#ifndef __CYGWIN__ + if (!HDstrncmp(dp->d_name, "lib", (size_t)3) && + (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { +#else + if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) && + HDstrstr(dp->d_name, ".dll") ) { +#endif + + h5_stat_t my_stat; + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s/%s", dir, dp->d_name); + + /* Get info for directory entry */ + if (HDstat(path, &my_stat) == -1) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno)) + + /* If it is a directory, skip it */ + if (S_ISDIR(my_stat.st_mode)) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } /* end if */ + } /* end while */ + +done: + if (dirp) + if (HDclosedir(dirp) < 0) + HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) + + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#else /* H5_HAVE_WIN32_API */ +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + WIN32_FIND_DATAA fdFile; + HANDLE hFind = INVALID_HANDLE_VALUE; + char *path = NULL; + char service[2048]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Specify a file mask. *.* = We want everything! */ + HDsprintf(service, "%s\\*.dll", dir); + if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") + + /* Loop over all the files */ + do { + /* Ignore '.' and '..' */ + if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { + + /* XXX: Probably just continue here and move the code below over one tab */ + + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName); + + /* Ignore directories */ + if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } + } while (FindNextFileA(hFind, &fdFile)); + +done: + if (hFind != INVALID_HANDLE_VALUE) + FindClose(hFind); + if (path) + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5PLpkg.h b/src/H5PLpkg.h index e356893..0d1c271 100644 --- a/src/H5PLpkg.h +++ b/src/H5PLpkg.h @@ -11,6 +11,12 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * Purpose: This file contains declarations which are visible only within + * the H5PL package. Source files outside the H5PL package should + * include H5PLprivate.h instead. + */ + #if !(defined H5PL_FRIEND || defined H5PL_MODULE) #error "Do not include this file outside the H5PL package!" #endif @@ -27,13 +33,92 @@ /* Package Private Macros */ /**************************/ -#define H5PL_MAX_PATH_NUM 16 +/* Whether to pre-load pathnames for plugin libraries */ +#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR + + +/****************************/ +/* Macros for supporting */ +/* both Windows and POSIX */ +/****************************/ + +/*******************/ +/* Windows support */ +/*******************/ +/* + * SPECIAL WINDOWS NOTE + * + * Some of the Win32 API functions expand to fooA or fooW depending on + * whether UNICODE or _UNICODE are defined. You MUST explicitly use + * the A version of the functions to force char * behavior until we + * work out a scheme for proper Windows Unicode support. + * + * If you do not do this, people will be unable to incorporate our + * source code into their own CMake builds if they define UNICODE. + */ +#ifdef H5_HAVE_WIN32_API + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ";" + + /* Handle for dynamic library */ +# define H5PL_HANDLE HINSTANCE + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) FreeLibrary(H) + + /* Clear error - nothing to do */ +# define H5PL_CLR_ERROR + + /* maximum size for expanding env vars */ +# define H5PL_EXPAND_BUFFER_SIZE 32767 + + typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); + +#else /* H5_HAVE_WIN32_API */ + + /*****************/ + /* POSIX support */ + /*****************/ + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ":" + + /* Handle for dynamic library */ +# define H5PL_HANDLE void * + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) dlclose(H) + + /* Clear error */ +# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) + + typedef const void *(*H5PL_get_plugin_info_t)(void); +#endif /* H5_HAVE_WIN32_API */ /****************************/ /* Package Private Typedefs */ /****************************/ +/* Data used to search for plugins */ +typedef struct H5PL_search_params_t { + H5PL_type_t type; + int id; +} H5PL_search_params_t; + /*****************************/ /* Package Private Variables */ @@ -44,5 +129,31 @@ /* Package Private Prototypes */ /******************************/ +/* Accessors to global variables and flags */ +H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/); +H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask); + +/* Plugin search and manipulation */ +H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/); +H5_DLL herr_t H5PL__close(H5PL_HANDLE handle); + +/* Plugin cache calls */ +H5_DLL herr_t H5PL__create_plugin_cache(void); +H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/); +H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle); +H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + +/* Plugin search path calls */ +H5_DLL herr_t H5PL__create_path_table(void); +H5_DLL herr_t H5PL__close_path_table(void); +H5_DLL unsigned H5PL__get_num_paths(void); +H5_DLL herr_t H5PL__append_path(const char *path); +H5_DLL herr_t H5PL__prepend_path(const char *path); +H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__remove_path(unsigned int index); +H5_DLL const char *H5PL__get_path(unsigned int index); +H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + #endif /* _H5PLpkg_H */ diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c new file mode 100644 index 0000000..30a0798 --- /dev/null +++ b/src/H5PLplugin_cache.c @@ -0,0 +1,308 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a plugin cache which stores information + * about plugins which have already been loaded. + * + * The plugin cache is implemented as a dynamic, global array which + * will grow as new plugins are added. The capacity of the cache + * never shrinks since plugins stay in memory once loaded. + * + * Note that this functionality has absolutely nothing to do with + * the metadata or chunk caches. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the plugin cache */ +#define H5PL_INITIAL_CACHE_CAPACITY 16 + +/* The amount to add to the capacity when the cache is full */ +#define H5PL_CACHE_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Type for the list of info for opened plugin libraries */ +typedef struct H5PL_plugin_t { + H5PL_type_t type; /* Plugin type */ + int id; /* ID for the plugin */ + H5PL_HANDLE handle; /* Plugin handle */ +} H5PL_plugin_t; + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__expand_cache(void); + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Cache for storing opened plugin libraries */ +static H5PL_plugin_t *H5PL_cache_g = NULL; + +/* The number of stored plugins */ +static unsigned int H5PL_num_plugins_g = 0; + +/* The capacity of the plugin cache */ +static unsigned int H5PL_cache_capacity_g = 0; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_plugin_cache + * + * Purpose: Create the cache that will store plugins that have already + * been loaded. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_plugin_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the plugin cache */ + H5PL_num_plugins_g = 0; + + H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY; + + if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache") + +done: + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_cache_g) + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_cache_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_plugin_cache + * + * Purpose: Close the cache of plugins that have already been loaded, + * closing all the plugins contained inside. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_plugin_cache(hbool_t *already_closed /*out*/) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + /* Close opened dynamic libraries */ + if (H5PL_cache_g) { + + /* Close any cached plugins */ + for (u = 0; u < H5PL_num_plugins_g; u++) + H5PL__close((H5PL_cache_g[u]).handle); + + /* Free the cache array */ + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_num_plugins_g = 0; + H5PL_cache_capacity_g = 0; + + /* Note that actually closed the table (needed by package close call) */ + *already_closed = FALSE; + } + else + *already_closed = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__close_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_cache + * + * Purpose: Expand the plugin cache when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed") + + /* Initialize the new memory */ + HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t)); + +done: + /* Set the cache capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__add_plugin + * + * Purpose: Add a plugin to the plugin cached. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Expand the cache if it is too small */ + if (H5PL_num_plugins_g >= H5PL_cache_capacity_g) + if (H5PL__expand_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache") + + /* Store the plugin info and bump the # of plugins */ + H5PL_cache_g[H5PL_num_plugins_g].type = type; + H5PL_cache_g[H5PL_num_plugins_g].id = id; + H5PL_cache_g[H5PL_num_plugins_g].handle = handle; + + H5PL_num_plugins_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__add_plugin() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_cache + * + * Purpose: Attempts to find a matching plugin from the cache. + * + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over all the plugins, looking for one that matches */ + for (u = 0; u < H5PL_num_plugins_g; u++) { + + /* If the plugin type (filter, etc.) and ID match, query the plugin for its info */ + if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) { + + H5PL_get_plugin_info_t get_plugin_info_function; + const H5Z_class2_t *filter_info; + + /* Get the "get plugin info" function from the plugin. + * + * See the other use of H5PL_GET_LIB_FUNC() for an explanation + * for why we disable -Wpedantic here. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info"))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") +#pragma GCC diagnostic pop + + /* Call the "get plugin info" function */ + if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)())) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Set output parameters */ + *found = TRUE; + *plugin_info = filter_info; + + /* No need to continue processing */ + break; + + } /* end if */ + + } /* end for */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_cache() */ + diff --git a/src/H5PLprivate.h b/src/H5PLprivate.h index 0ab8f8c..bc12e64 100644 --- a/src/H5PLprivate.h +++ b/src/H5PLprivate.h @@ -10,8 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Raymond Lu - * 13 February 2013 +/* + * This file contains private information about the H5PL module */ #ifndef _H5PLprivate_H diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h index 9ce1fca..3b36ccd 100644 --- a/src/H5PLpublic.h +++ b/src/H5PLpublic.h @@ -10,8 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Raymond Lu - * 13 February 2013 +/* + * This file contains public declarations for the H5PL module. */ #ifndef _H5PLpublic_H @@ -24,31 +24,35 @@ /* Public Typedefs */ /*******************/ +/* Special string to indicate no plugin loading. + */ +#define H5PL_NO_PLUGIN "::" + /* Plugin type used by the plugin library */ typedef enum H5PL_type_t { - H5PL_TYPE_ERROR = -1, /*error */ - H5PL_TYPE_FILTER = 0, /*filter */ - H5PL_TYPE_NONE = 1 /*this must be last! */ + H5PL_TYPE_ERROR = -1, /* Error */ + H5PL_TYPE_FILTER = 0, /* Filter */ + H5PL_TYPE_NONE = 1 /* This must be last! */ } H5PL_type_t; /* Common dynamic plugin type flags used by the set/get_loading_state functions */ -#define H5PL_FILTER_PLUGIN 0x0001 -#define H5PL_ALL_PLUGIN 0xFFFF +#define H5PL_FILTER_PLUGIN 0x0001 +#define H5PL_ALL_PLUGIN 0xFFFF #ifdef __cplusplus extern "C" { #endif /* plugin state */ -H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type); -H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/); -H5_DLL herr_t H5PLappend(const char *plugin_path); -H5_DLL herr_t H5PLprepend(const char *plugin_path); -H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index); -H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index); +H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask); +H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/); +H5_DLL herr_t H5PLappend(const char *search_path); +H5_DLL herr_t H5PLprepend(const char *search_path); +H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index); +H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index); H5_DLL herr_t H5PLremove(unsigned int index); -H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size); -H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/); +H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size); +H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/); #ifdef __cplusplus } diff --git a/src/H5system.c b/src/H5system.c index 7e25540..1e718e7 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1237,3 +1237,53 @@ H5_get_time(void) } /* end H5_get_time() */ +#ifdef H5_HAVE_WIN32_API + +#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767 + + +/*------------------------------------------------------------------------- + * Function: H5_expand_windows_env_vars() + * + * Purpose: Replaces windows environment variables of the form %foo% + * with user-specific values. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_expand_windows_env_vars(char **env_var) +{ + long n_chars = 0; + char *temp_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate buffer for expanded environment variable string */ + if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") + + /* Expand the environment variable string */ + if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE) { + temp_buf = (char *)H5MM_xfree(temp_buf); + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") + } + + if (n_chars == 0) { + temp_buf = (char *)H5MM_xfree(temp_buf); + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") + } + + *env_var = (char *)H5MM_xfree(*env_var); + *env_var = temp_buf; + +done: + if (FAIL == ret_value && temp_buf) + temp_buf = (char *)H5MM_xfree(temp_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_expand_windows_env_vars() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 97f7179..4522228 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -119,6 +119,7 @@ extern "C" { H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem); + H5_DLL herr_t H5_expand_windows_env_vars(char **env_var); /* Round functions only needed for VS2012 and earlier. * They are always built to ensure they don't go stale and diff --git a/src/Makefile.am b/src/Makefile.am index 0b664a7..9a64717 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ H5PB.c \ - H5PL.c \ + H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \ H5R.c H5Rdeprec.c \ H5UC.c \ H5RS.c \ diff --git a/test/plugin.c b/test/plugin.c index 1254e9a..f536a9e 100644 --- a/test/plugin.c +++ b/test/plugin.c @@ -10,9 +10,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu - * 13 February 2013 - * * Purpose: Tests the plugin module (H5PL) */ @@ -660,13 +657,10 @@ error: /*------------------------------------------------------------------------- * Function: test_groups_with_filters * - * Purpose: Tests opening group with dynamically loaded filters - * - * Return: Success: 0 - * Failure: -1 + * Purpose: Tests opening group with dynamically loaded filters * - * Programmer: Raymond Lu - * 1 April 2013 + * Return: Success: 0 + * Failure: -1 * *------------------------------------------------------------------------- */ @@ -710,261 +704,380 @@ error: /*------------------------------------------------------------------------- - * Function: test_filter_path_apis + * Function: test_path_api_calls * - * Purpose: Tests accessing the path table for dynamically loaded filters + * Purpose: Tests the H5PL API calls that manipulate the plugin search + * paths. * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL + * *------------------------------------------------------------------------- */ static herr_t -test_filter_path_apis(void) +test_path_api_calls(void) { - herr_t ret_value = -1; - unsigned int i; - unsigned int ndx; + unsigned int n_starting_paths; + unsigned int u; + unsigned int n_paths; herr_t ret; - ssize_t pathlen = -1; - char pathname[256]; - char tempname[256]; + ssize_t path_len = -1; + char path[256]; + char temp_name[256]; + herr_t ret_value = -1; HDputs("Testing access to the filter path table"); - if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) TEST_ERROR + if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) + TEST_ERROR + + /* Set the number of paths to create for this test. + * + * This should be set high enough to ensure that at least one array + * expansion will take place. See H5PLpath.c for details. + */ + n_starting_paths = 42; + - H5PLsize(&ndx); + /****************/ + /* H5PLremove() */ + /****************/ + /* Remove all the current paths */ TESTING(" remove"); - /* Remove all existing paths*/ - for(i=ndx; i > 0; i--) - if(H5PLremove(i-1) < 0) { - HDfprintf(stderr," at %d: %s\n", i, pathname); + + /* Get the current size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + + /* Remove all existing paths */ + for(u = n_paths; u > 0; u--) + if(H5PLremove(u-1) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); TEST_ERROR - } /* end if */ + } + /* Verify the table is empty */ - H5PLsize(&ndx); - if(ndx > 0) TEST_ERROR + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths > 0) + TEST_ERROR + PASSED(); - TESTING(" remove (exceed min)"); - /* Exceed the min path removal */ + + TESTING(" remove (index 0 in empty table)"); + + /* Try to remove index zero in an empty list (SHOULD FAIL) */ H5E_BEGIN_TRY { ret = H5PLremove(0); } H5E_END_TRY - if(ret >= 0) TEST_ERROR + if(ret >= 0) + TEST_ERROR + PASSED(); + + /****************/ + /* H5PLappend() */ + /****************/ + TESTING(" append"); - /* Create multiple paths to fill table */ - for(i=0; i < H5PL_MAX_PATH_NUM; i++) { - HDsprintf(pathname, "a_path_%d", i); - if(H5PLappend(pathname) < 0) { - HDfprintf(stderr," at %d: %s\n", i, pathname); + + /* Add a bunch of paths to the path table */ + for(u = 0; u < n_starting_paths; u++) { + HDsprintf(path, "a_path_%u", u); + if(H5PLappend(path) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); TEST_ERROR } } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR - PASSED(); - TESTING(" append (exceed)"); - /* Exceed the max path append */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM); - ret = H5PLappend(pathname); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR PASSED(); - TESTING(" remove (exceed max)"); - /* Exceed the max path removal */ + + /**********************/ + /* H5PLremove() again */ + /**********************/ + + TESTING(" remove (index too high)"); + + /* Try to remove a path where the index is beyond the table capacity (SHOULD FAIL) */ H5E_BEGIN_TRY { - ret = H5PLremove(H5PL_MAX_PATH_NUM); + ret = H5PLremove(n_starting_paths); } H5E_END_TRY - if(ret >= 0) TEST_ERROR + + if(ret >= 0) + TEST_ERROR + PASSED(); + + /*************/ + /* H5PLget() */ + /*************/ + TESTING(" get (path name)"); - if((pathlen = H5PLget(0, NULL, 0)) <= 0) { + + /* Get the path length by passing in NULL */ + if((path_len = H5PLget(0, NULL, 0)) <= 0) { HDfprintf(stderr," get path 0 length failed\n"); TEST_ERROR } - if(pathlen != 8) TEST_ERROR + if(path_len != 8) + TEST_ERROR - if((pathlen = H5PLget(0, pathname, 256)) <= 0) { - HDfprintf(stderr," get 0 len: %d : %s\n", pathlen, pathname); + /* Get the path */ + if((path_len = H5PLget(0, path, 256)) <= 0) { + HDfprintf(stderr," get 0 len: %u : %s\n", path_len, path); TEST_ERROR } - if(HDstrcmp(pathname, "a_path_0") != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + if(HDstrcmp(path, "a_path_0") != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } + PASSED(); - TESTING(" get (bounds)"); - if((pathlen = H5PLget(1, pathname, 256)) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_1") != 0) { - HDfprintf(stderr," get 1: %s\n", pathname); + + TESTING(" get (high and low indices)"); + + /* Get path at index 1 */ + if((path_len = H5PLget(1, path, 256)) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_1") != 0) { + HDfprintf(stderr," get 1: %s\n", path); TEST_ERROR } - if((pathlen = H5PLget(H5PL_MAX_PATH_NUM - 1, pathname, 256)) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM - 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get %d: %s\n", H5PL_MAX_PATH_NUM - 1, pathname); + + /* Get path at the last index */ + if((path_len = H5PLget(n_starting_paths - 1, path, 256)) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%u", n_starting_paths - 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get %u: %s\n", n_starting_paths - 1, path); TEST_ERROR } + PASSED(); - TESTING(" get (bounds exceed)"); + + TESTING(" get (index too high)"); + + /* Get path at the last + 1 index (SHOULD FAIL) */ H5E_BEGIN_TRY { - pathlen = H5PLget(H5PL_MAX_PATH_NUM, NULL, 0); + path_len = H5PLget(n_starting_paths, NULL, 0); } H5E_END_TRY - if(pathlen > 0) TEST_ERROR + if(path_len > 0) + TEST_ERROR + PASSED(); - TESTING(" remove (verify for prepend)"); - /* Remove one path*/ - if(H5PLremove(8) < 0) TEST_ERROR + + /*****************/ + /* H5PLprepend() */ + /*****************/ + + /* We'll remove a path at an arbitrary index and then + * prepend a new path. + */ + + TESTING(" remove (arbitrary index 1)"); + + /* Remove one path */ + if(H5PLremove(8) < 0) + TEST_ERROR /* Verify that the entries were moved */ - if((pathlen = H5PLget(8, pathname, 256)) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_9") != 0) { - HDfprintf(stderr," get 8: %s\n", pathname); + if((path_len = H5PLget(8, path, 256)) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_9") != 0) { + HDfprintf(stderr," get 8: %s\n", path); TEST_ERROR } + + /* Verify the table shrank */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths - 1) + TEST_ERROR + PASSED(); - /* Verify the table is not full */ - H5PLsize(&ndx); - if (ndx != H5PL_MAX_PATH_NUM - 1) TEST_ERROR TESTING(" prepend"); - /* Prepend one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(H5PLprepend(pathname) < 0) { - HDfprintf(stderr," prepend %d: %s\n", H5PL_MAX_PATH_NUM + 1, pathname); + + /* Prepend one path */ + HDsprintf(path, "a_path_%d", n_starting_paths + 1); + if(H5PLprepend(path) < 0) { + HDfprintf(stderr," prepend %u: %s\n", n_starting_paths + 1, path); TEST_ERROR } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR + /* Verify the table increased */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR /* Verify that the entries were moved */ - if(H5PLget(8, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_7") != 0) { - HDfprintf(stderr," get 8: %s\n", pathname); + if(H5PLget(8, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_7") != 0) { + HDfprintf(stderr," get 8: %s\n", path); TEST_ERROR } - if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + + /* Verify that the path was inserted at index zero */ + if(H5PLget(0, path, 256) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%d", n_starting_paths + 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } - PASSED(); - TESTING(" prepend (exceed)"); - /* Exceed the max path prepend */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 2); - ret = H5PLprepend(pathname); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR PASSED(); + + /*****************/ + /* H5PLreplace() */ + /*****************/ + TESTING(" replace"); - /* Replace one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 4); - if(H5PLreplace(pathname, 1) < 0) { - HDfprintf(stderr," replace 1: %s\n", pathname); + + /* Replace one path at index 1 */ + HDsprintf(path, "a_path_%u", n_starting_paths + 4); + if(H5PLreplace(path, 1) < 0) { + HDfprintf(stderr," replace 1: %s\n", path); TEST_ERROR } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR + /* Verify the table size remained the same */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR + + /* Verify that the entries were not moved by + * inspecting the paths at indices +/- 1. + */ - /* Verify that the entries were not moved */ - if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + /* Check path at index 0 */ + if(H5PLget(0, path, 256) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%u", n_starting_paths + 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } - if(H5PLget(2, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_1") != 0) { - HDfprintf(stderr," get 2: %s\n", pathname); + + /* Check path at index 2 */ + if(H5PLget(2, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_1") != 0) { + HDfprintf(stderr," get 2: %s\n", path); TEST_ERROR } + PASSED(); - TESTING(" remove (verify for insert)"); - /* Remove one path*/ - if(H5PLremove(4) < 0) TEST_ERROR + + /****************/ + /* H5PLinsert() */ + /****************/ + + /* We'll remove a path at an arbitrary index and then + * insert a new path. + */ + + TESTING(" remove (arbitrary index 2)"); + + /* Remove one path */ + if(H5PLremove(4) < 0) + TEST_ERROR /* Verify that the entries were moved */ - if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_4") != 0) { - HDfprintf(stderr," get 4: %s\n", pathname); + if(H5PLget(4, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_4") != 0) { + HDfprintf(stderr," get 4: %s\n", path); TEST_ERROR } + + /* Verify the table size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths - 1) + TEST_ERROR PASSED(); - /* Verify the table is not full */ - H5PLsize(&ndx); - if(ndx != 15) TEST_ERROR TESTING(" insert"); - /* Insert one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 5); - if(H5PLinsert(pathname, 3) < 0) { - HDfprintf(stderr," insert 3: %s\n", pathname); + + /* Insert one path at index 3*/ + HDsprintf(path, "a_path_%d", n_starting_paths + 5); + if(H5PLinsert(path, 3) < 0) { + HDfprintf(stderr," insert 3: %s\n", path); TEST_ERROR } /* Verify that the entries were moved */ - if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_2") != 0) { - HDfprintf(stderr," get 4: %s\n", pathname); + if(H5PLget(4, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_2") != 0) { + HDfprintf(stderr," get 4: %s\n", path); TEST_ERROR } + + /* Verify the table size increased */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR + PASSED(); - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR - TESTING(" insert (exceed)"); - /* Exceed the max path insert */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 6); - ret = H5PLinsert(pathname, 12); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR + /****************/ + /* H5PLremove() */ + /****************/ + + /* Remove all the current paths */ + TESTING(" remove (all)"); + + /* Get the current size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + + /* Remove all existing paths */ + for(u = n_paths; u > 0; u--) + if(H5PLremove(u-1) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); + TEST_ERROR + } + + /* Verify the table is empty */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths > 0) + TEST_ERROR PASSED(); - ret_value = 0; + + return SUCCEED; error: - return ret_value; -} + return FAIL; +} /* end test_path_api_calls() */ /*------------------------------------------------------------------------- * Function: main * - * Purpose: Tests the plugin module (H5PL) - * - * Return: Success: exit(EXIT_SUCCESS) + * Purpose: Tests the plugin module (H5PL) * - * Failure: exit(EXIT_FAILURE) - * - * Programmer: Raymond Lu - * 14 March 2013 + * Return: EXIT_SUCCESS/EXIT_FAILURE * *------------------------------------------------------------------------- */ @@ -1063,18 +1176,19 @@ main(void) if(H5Fclose(file) < 0) TEST_ERROR /* Test the APIs for access to the filter plugin path table */ - nerrors += (test_filter_path_apis() < 0 ? 1 : 0); + nerrors += (test_path_api_calls() < 0 ? 1 : 0); - if(nerrors) TEST_ERROR + if(nerrors) + TEST_ERROR HDprintf("All plugin tests passed.\n"); h5_cleanup(FILENAME, fapl); - return 0; + HDexit(EXIT_SUCCESS); error: nerrors = MAX(1, nerrors); HDprintf("***** %d PLUGIN TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ -- cgit v0.12 From 274f71be174c431be6e94b1d4f4c100ff090e749 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 14 Jul 2017 12:58:29 -0700 Subject: Fixed a failing Java plugin test. --- java/test/TestH5PL.java | 88 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/java/test/TestH5PL.java b/java/test/TestH5PL.java index aa59478..8ce708b 100644 --- a/java/test/TestH5PL.java +++ b/java/test/TestH5PL.java @@ -70,26 +70,74 @@ public class TestH5PL { @Test public void TestH5PLpaths() { try { - int original_entries = H5.H5PLsize(); - H5.H5PLappend("path_one"); - int plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+1) == plugin_entries); - H5.H5PLprepend("path_two"); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries); - H5.H5PLinsert("path_three", original_entries); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+3) == plugin_entries); - String first_path = H5.H5PLget(original_entries); - assertTrue("First path was : "+first_path + " ",first_path.compareToIgnoreCase("path_three")==0); - H5.H5PLreplace("path_four", original_entries); - first_path = H5.H5PLget(original_entries); - assertTrue("First path changed to : "+first_path + " ",first_path.compareToIgnoreCase("path_four")==0); - H5.H5PLremove(original_entries); - first_path = H5.H5PLget(original_entries); - assertTrue("First path now : "+first_path + " ",first_path.compareToIgnoreCase("path_two")==0); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries); + // Get the original number of paths + int nStartPaths = H5.H5PLsize(); + + int nPaths; /* # paths from H5PLSize() */ + int nTruePaths = nStartPaths; /* What the # paths should be */ + int index; /* Path table index */ + String path; /* Path from H5PLget() */ + + // APPEND a path and ensure it was added correctly + String pathAppend = "path_append"; + H5.H5PLappend(pathAppend); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + index = nTruePaths - 1; + path = H5.H5PLget(index); + assertTrue("Path should be " + pathAppend + " but was " + path, path.compareToIgnoreCase(pathAppend) == 0); + + // PREPEND a path and ensure it was added correctly + String pathPrepend = "path_prepend"; + H5.H5PLprepend(pathPrepend); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + index = 0; + path = H5.H5PLget(index); + assertTrue("Path should be " + pathPrepend + " but was " + path, path.compareToIgnoreCase(pathPrepend) == 0); + + // INSERT a path and ensure it was added correctly + // Inserting at the index == # of start paths ensures we're in the middle + String pathInsert = "path_insert"; + index = nStartPaths; + H5.H5PLinsert(pathInsert, index); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathInsert + " but was " + path, path.compareToIgnoreCase(pathInsert) == 0); + + // REPLACE the path we just added and ensure it updated correctly + String pathReplace = "path_replace"; + index = nStartPaths; + H5.H5PLreplace(pathReplace, index); + + nPaths = H5.H5PLsize(); + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathReplace + " but was " + path, path.compareToIgnoreCase(pathReplace) == 0); + + // REMOVE the path we just replaced and check that the table was compacted + // The (index+1) path should move down to fill the space when the path is removed. + index = nStartPaths; + String pathRemove = H5.H5PLget(index + 1); + H5.H5PLremove(index); + + nPaths = H5.H5PLsize(); + nTruePaths--; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathRemove + " but was " + path, path.compareToIgnoreCase(pathRemove) == 0); } catch (Throwable err) { err.printStackTrace(); -- cgit v0.12 From ee7816bbfb3f22d42d5db7d779efe54ae29528cb Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jul 2017 13:24:01 -0500 Subject: Stripped C standard library headers (and a couple of macros) from the tools code. This is all handled in H5private.h, which should be included in all tools source files. --- tools/lib/h5diff.c | 2 -- tools/lib/h5tools.c | 6 ------ tools/lib/h5tools_dump.c | 6 ------ tools/lib/h5tools_ref.c | 2 -- tools/lib/h5tools_str.c | 8 -------- tools/lib/h5tools_utils.c | 10 ---------- tools/lib/io_timer.c | 3 --- tools/src/h5copy/h5copy.c | 2 -- tools/src/h5diff/h5diff_common.c | 2 -- tools/src/h5diff/h5diff_main.c | 3 --- tools/src/h5diff/ph5diff_main.c | 3 --- tools/src/h5dump/h5dump.c | 2 -- tools/src/h5dump/h5dump_ddl.c | 2 -- tools/src/h5dump/h5dump_xml.c | 2 -- tools/src/h5repack/h5repack.c | 4 ---- tools/src/h5stat/h5stat.c | 2 -- tools/src/misc/h5mkgrp.c | 2 -- tools/src/misc/h5repart.c | 34 ++-------------------------------- 18 files changed, 2 insertions(+), 93 deletions(-) diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index afb36d9..9da5b6b 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include - #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 3729cac..5031e44 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -12,16 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke - * Thursday, July 23, 1998 - * * Purpose: A library for displaying the values of a dataset in a human * readable format. */ -#include -#include - #include "h5tools.h" #include "h5tools_dump.h" #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index 1a57512..fb79b77 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -12,16 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke - * Thursday, July 23, 1998 - * * Purpose: A library for displaying the values of a dataset in a human * readable format. */ -#include -#include - #include "h5tools.h" #include "h5tools_dump.h" #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_ref.c b/tools/lib/h5tools_ref.c index 85850e3..e000080 100644 --- a/tools/lib/h5tools_ref.c +++ b/tools/lib/h5tools_ref.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "h5tools_ref.h" #include "H5private.h" #include "H5SLprivate.h" diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index fa15785..a66cfe5 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -12,16 +12,8 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Bill Wendling - * Monday, 19. February 2001 - * * Purpose: These are string functions for us to use and abuse. */ -#include -#include -#include -#include - #include "H5private.h" #include "h5tools.h" /* for h5tool_format_t structure */ #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index c361e25..08213df 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -12,20 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Bill Wendling - * Tuesday, 6. March 2001 - */ - -/* * Portions of this work are derived from _Obfuscated C and Other Mysteries_, * by Don Libes, copyright (c) 1993 by John Wiley & Sons, Inc. */ -#include -#include -#include -#include - #include "h5tools.h" #include "h5tools_utils.h" #include "H5private.h" diff --git a/tools/lib/io_timer.c b/tools/lib/io_timer.c index e3318e9..4aa2195 100644 --- a/tools/lib/io_timer.c +++ b/tools/lib/io_timer.c @@ -22,9 +22,6 @@ * This is a module of useful timing functions for performance testing. */ -#include -#include - #include "H5private.h" #include "hdf5.h" diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c index 390b93e..3f91fce 100644 --- a/tools/src/h5copy/h5copy.c +++ b/tools/src/h5copy/h5copy.c @@ -14,8 +14,6 @@ #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" -#include -#include /* Name of tool */ #define PROGRAMNAME "h5copy" diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index 0537b9f..1069a31 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "H5private.h" #include "h5diff.h" #include "h5diff_common.h" diff --git a/tools/src/h5diff/h5diff_main.c b/tools/src/h5diff/h5diff_main.c index 92a1610..66ff71e 100644 --- a/tools/src/h5diff/h5diff_main.c +++ b/tools/src/h5diff/h5diff_main.c @@ -11,9 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include -#include #include "H5private.h" #include "h5diff.h" #include "h5diff_common.h" diff --git a/tools/src/h5diff/ph5diff_main.c b/tools/src/h5diff/ph5diff_main.c index bfeb408..192067f 100644 --- a/tools/src/h5diff/ph5diff_main.c +++ b/tools/src/h5diff/ph5diff_main.c @@ -11,9 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include -#include #include "H5private.h" #include "h5diff.h" #include "ph5diff.h" diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index b53c212..bf2e127 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "h5dump.h" #include "h5dump_ddl.h" diff --git a/tools/src/h5dump/h5dump_ddl.c b/tools/src/h5dump/h5dump_ddl.c index 8ce6cd6..0a45840 100644 --- a/tools/src/h5dump/h5dump_ddl.c +++ b/tools/src/h5dump/h5dump_ddl.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "H5private.h" #include "h5tools.h" diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index 1c3978d..bb0fd7a 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "H5private.h" #include "h5tools.h" diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index bc8527b..4860d9e 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -11,10 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include -#include - #include "H5private.h" #include "h5repack.h" #include "h5tools.h" diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 6aee7a8..6f196b4 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include #include "H5private.h" /* Generic Functions */ #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index 597b6b3..43d6bfe 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -15,8 +15,6 @@ #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" -#include -#include /* Name of tool */ #define PROGRAMNAME "h5mkgrp" diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c index 4432621..cdc554f 100644 --- a/tools/src/misc/h5repart.c +++ b/tools/src/misc/h5repart.c @@ -25,39 +25,9 @@ /* See H5private.h for how to include system headers */ #include "hdf5.h" #include "H5private.h" -#ifdef H5_STDC_HEADERS -# include -# include -# include -# include -# include -# include -#endif - -#ifdef H5_HAVE_UNISTD_H -# include -# include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -# include -#endif - -#ifndef FALSE -# define FALSE 0 -#endif -#ifndef TRUE -# define TRUE 1 -#endif -# define NAMELEN 4096 -#define GB *1024*1024*1024 -#ifndef MIN -# define MIN(X,Y) ((X)<(Y)?(X):(Y)) -#endif -#ifndef MIN3 -# define MIN3(X,Y,Z) MIN(MIN(X,Y),Z) -#endif +#define NAMELEN 4096 +#define GB *1024*1024*1024 /*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. *The first one updates the member file size in the superblock. The second one -- cgit v0.12 From 66efce85e7ad9430d5e58d529d7b7a55b0d4ac5f Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Fri, 9 Jun 2017 14:01:07 -0500 Subject: Changes made based on RFC review comments Test the changes in a branch via daily testing. --- src/H5Fprivate.h | 4 +++- src/H5MF.c | 2 +- src/H5MFsection.c | 32 ++++++++++++++++---------------- src/H5Pfcpl.c | 3 +++ test/tfile.c | 17 +++++++++++++++++ 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 6f68a62..eba48de 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -549,9 +549,11 @@ #define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 /* For paged aggregation: minimum value for file space page size */ #define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 +/* For paged aggregation: maxiumum value for file space page size: 1 gigabyte */ +#define H5F_FILE_SPACE_PAGE_SIZE_MAX 1024*1024*1024 /* For paged aggregation: drop free-space with size <= this threshold for small meta section */ -#define H5F_FILE_SPACE_PGEND_META_THRES 10 +#define H5F_FILE_SPACE_PGEND_META_THRES 0 /* Default for threshold for alignment (can be set via H5Pset_alignment()) */ #define H5F_ALIGN_DEF 1 diff --git a/src/H5MF.c b/src/H5MF.c index e54d809..d7af56a 100644 --- a/src/H5MF.c +++ b/src/H5MF.c @@ -1519,7 +1519,7 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ H5F_mem_page_t fs_type; /* Free space type */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 02fc2d9..14e0ad1 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -130,27 +130,27 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Class info for "small" free space sections */ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ /* Class variables */ - H5MF_FSPACE_SECT_SMALL, /* Section type */ - 0, /* Extra serialized size */ - H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ - NULL, /* Class private info */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ /* Class methods */ - NULL, /* Initialize section class */ - NULL, /* Terminate section class */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ /* Object methods */ - H5MF_sect_small_add, /* Add section */ - NULL, /* Serialize section */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ H5MF_sect_deserialize, /* Deserialize section */ - H5MF_sect_small_can_merge, /* Can sections merge? */ - H5MF_sect_small_merge, /* Merge sections */ - H5MF_sect_small_can_shrink, /* Can section shrink container?*/ - H5MF_sect_small_shrink, /* Shrink container w/section */ - H5MF_sect_free, /* Free section */ - H5MF_sect_valid, /* Check validity of section */ - H5MF_sect_split, /* Split section node for alignment */ - NULL, /* Dump debugging for section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + NULL, /* Can section shrink container?*/ + NULL, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ }}; /* Class info for "large" free space sections */ diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index 5383aae..6b0d2c0 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -1472,6 +1472,9 @@ H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + if(fsp_size > H5F_FILE_SPACE_PAGE_SIZE_MAX) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to more than 1GB") + /* Set the value*/ if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") diff --git a/test/tfile.c b/test/tfile.c index 533bb24..027ad62 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -109,6 +109,7 @@ #define TEST_THRESHOLD10 10 /* Free space section threshold */ #define FSP_SIZE_DEF 4096 /* File space page size default */ #define FSP_SIZE512 512 /* File space page size */ +#define FSP_SIZE1G 1024*1024*1024 /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ @@ -3581,6 +3582,9 @@ test_filespace_info(const char *env_h5_drvr) * Setting value less than 512 will return an error; * --setting file space page size to 0 * --setting file space page size to 511 + * + * File space page size has a maximum size of 1 gigabyte. + * Setting value greater than 1 gigabyte will return an error. */ /* Create file creation property list template */ fcpl = H5Pcreate(H5P_FILE_CREATE); @@ -3598,6 +3602,12 @@ test_filespace_info(const char *env_h5_drvr) } H5E_END_TRY; VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 1GB+1: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G+1); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 512: should succeed */ ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); CHECK(ret, FAIL, "H5Pset_file_space_page_size"); @@ -3605,6 +3615,13 @@ test_filespace_info(const char *env_h5_drvr) CHECK(ret, FAIL, "H5Pget_file_space_page_size"); VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + /* Setting to 1GB: should succeed */ + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE1G, "H5Pget_file_space_page_size"); + /* Close property list */ H5Pclose(fcpl); -- cgit v0.12 From ba17f16e4ca2ab92160a40d54f570042a2c9ea14 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Mon, 10 Jul 2017 02:40:59 -0500 Subject: Skip test in test/fheap.c when: a) multi/split drivers and b) persisting free-space or using paged aggregation strategy because the library will fail file creation (temporary) for the above conditions. --- src/H5MFsection.c | 2 +- test/fheap.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 14e0ad1..02e3218 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -674,7 +674,7 @@ HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info. HDfprintf(stderr, "%s: section is dropped\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ - /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + /* Adjust the section if it is not at page end but its size + prem is at page end */ else if(prem <= H5F_PGEND_META_THRES(udata->f)) { (*sect)->sect_info.size += prem; diff --git a/test/fheap.c b/test/fheap.c index 4be6cb9..6c3a8ac 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -16377,6 +16377,16 @@ main(void) unsigned nerrors = 0; /* Cumulative error count */ unsigned num_pb_fs = 1; /* The number of settings to test for page buffering and file space handling */ int ExpressMode; /* Express testing level */ + const char *envval; /* Environment variable */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + + /* Don't run this test using certain file drivers */ + envval = HDgetenv("HDF5_DRIVER"); + if(envval == NULL) + envval = "nomatch"; + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi")); /* Reset library */ h5_reset(); @@ -16428,6 +16438,12 @@ main(void) shared_wobj_g[u] = (unsigned char)u; for(v = 0; v < num_pb_fs; v++) { + /* Skip test when: + a) multi/split drivers and + b) persisting free-space or using paged aggregation strategy + because the library will fail file creation (temporary) for the above conditions */ + if(!contig_addr_vfd && v) + break; if((fcpl = H5Pcopy(def_fcpl)) < 0) TEST_ERROR -- cgit v0.12 From ffc9b1d917c88f8c3e51ee1c2cb6924bb07328a7 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 19 Jul 2017 10:46:27 -0500 Subject: Stripped C standard library headers from tools test code. --- tools/lib/io_timer.c | 4 ---- tools/lib/ph5diff.h | 4 ---- tools/test/h5copy/h5copygentest.c | 1 - tools/test/h5dump/h5dumpgentest.c | 1 - tools/test/h5import/h5importtest.c | 1 - tools/test/h5jam/h5jamgentest.c | 2 -- tools/test/h5jam/tellub.c | 6 ------ tools/test/h5repack/testh5repack_detect_szip.c | 1 - tools/test/misc/talign.c | 3 --- 9 files changed, 23 deletions(-) diff --git a/tools/lib/io_timer.c b/tools/lib/io_timer.c index 4aa2195..a6885df 100644 --- a/tools/lib/io_timer.c +++ b/tools/lib/io_timer.c @@ -25,10 +25,6 @@ #include "H5private.h" #include "hdf5.h" -#ifdef H5_HAVE_PARALLEL -#include -#endif - #include "io_timer.h" /* diff --git a/tools/lib/ph5diff.h b/tools/lib/ph5diff.h index 9628d45..996a611 100644 --- a/tools/lib/ph5diff.h +++ b/tools/lib/ph5diff.h @@ -42,9 +42,5 @@ struct diffs_found int not_cmp; }; -#ifdef H5_HAVE_PARALLEL -#include -#endif - #endif /* _PH5DIFF_H__ */ diff --git a/tools/test/h5copy/h5copygentest.c b/tools/test/h5copy/h5copygentest.c index d4d6a08..35f9132 100644 --- a/tools/test/h5copy/h5copygentest.c +++ b/tools/test/h5copy/h5copygentest.c @@ -14,7 +14,6 @@ /* * Generate the binary hdf5 file for the h5copy tests */ -#include #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 44c4369..2128acb 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -20,7 +20,6 @@ * trying it on a new platform, ...), you need to verify the correctness * of the expected output and update the corresponding *.ddl files. */ -#include #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5import/h5importtest.c b/tools/test/h5import/h5importtest.c index 00ae2e7..489bc01 100644 --- a/tools/test/h5import/h5importtest.c +++ b/tools/test/h5import/h5importtest.c @@ -11,7 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include #include "H5private.h" #ifdef H5_HAVE_WIN32_API diff --git a/tools/test/h5jam/h5jamgentest.c b/tools/test/h5jam/h5jamgentest.c index d713bb9..9f3d000 100644 --- a/tools/test/h5jam/h5jamgentest.c +++ b/tools/test/h5jam/h5jamgentest.c @@ -20,8 +20,6 @@ * trying it on a new platform, ...), you need to verify the correctness * of the expected output and update the corresponding *.ddl files. */ -#include -#include #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5jam/tellub.c b/tools/test/h5jam/tellub.c index fad14b7..8e4b3ac 100644 --- a/tools/test/h5jam/tellub.c +++ b/tools/test/h5jam/tellub.c @@ -11,12 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include - -#ifdef H5_HAVE_UNISTD_H -#include -#endif - #include "hdf5.h" #include "H5private.h" #include "h5tools.h" diff --git a/tools/test/h5repack/testh5repack_detect_szip.c b/tools/test/h5repack/testh5repack_detect_szip.c index e08d5ab..6e7a24e 100644 --- a/tools/test/h5repack/testh5repack_detect_szip.c +++ b/tools/test/h5repack/testh5repack_detect_szip.c @@ -11,7 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include #include "h5repack.h" #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/test/misc/talign.c b/tools/test/misc/talign.c index 9a72557..ce866b4 100644 --- a/tools/test/misc/talign.c +++ b/tools/test/misc/talign.c @@ -15,9 +15,6 @@ * Small program to illustrate the "misalignment" of members within a compound * datatype, in a datatype fixed by H5Tget_native_type(). */ -#include -#include -/*#include *//* Required for unlink() */ #include "hdf5.h" #include "H5private.h" -- cgit v0.12 From ec7450c4b4b44a61b282535b981576d888286fca Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 19 Jul 2017 15:26:34 -0500 Subject: Closed a wayward fapl ID in tools/test/misc/repart_test.c and tidied. --- tools/test/misc/repart_test.c | 95 +++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 53 deletions(-) diff --git a/tools/test/misc/repart_test.c b/tools/test/misc/repart_test.c index 372f46a..4016ee8 100644 --- a/tools/test/misc/repart_test.c +++ b/tools/test/misc/repart_test.c @@ -12,19 +12,16 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu - * June 1, 2005 - * * Purpose: This program tests family files after being repartitioned * by h5repart. It simply tries to reopen the files with * correct family driver and member size. */ #include "hdf5.h" +#include "H5private.h" #define KB 1024 #define FAMILY_H5REPART_SIZE1 20000 #define FAMILY_H5REPART_SIZE2 (5*KB) -#define MAX(a,b) (a>b ? a:b) const char *FILENAME[] = { "fst_family%05d.h5", @@ -42,52 +39,54 @@ herr_t test_sec2_h5repart_opens(void); * * Purpose: Tries to reopen family files. * - * Return: Success: exit(0) - * - * Failure: exit(1) + * Return: SUCCEED/FAIL * - * Programmer: Raymond Lu - * June 1, 2005 - * - * Modifications: *------------------------------------------------------------------------- */ herr_t test_family_h5repart_opens(void) { - hid_t file=(-1), fapl=(-1); + hid_t fid = -1; + hid_t fapl_id = -1; /* open 1st file(single member file) with correct family size(20000 byte) */ - if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; - if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT)<0) + if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT) < 0) goto error; - if((file=H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl))<0) + if ((fid = H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl_id))<0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; /* open 2nd file(multiple member files) with correct family size(5KB) */ - if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT)<0) + if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT) < 0) + goto error; + + if ((fid = H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl_id)) < 0) goto error; - if((file=H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl))<0) + if (H5Pclose(fapl_id) < 0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Pclose(fapl_id); + H5Fclose(fid); } H5E_END_TRY; - return -1; -} + + return FAIL; + +} /* end test_family_h5repart_opens() */ + /*------------------------------------------------------------------------- @@ -95,36 +94,32 @@ error: * * Purpose: Tries to reopen a sec2 file. * - * Return: Success: exit(0) + * Return: SUCCEED/FAIL * - * Failure: exit(1) - * - * Programmer: Raymond Lu - * June 21, 2005 - * - * Modifications: *------------------------------------------------------------------------- */ herr_t test_sec2_h5repart_opens(void) { - hid_t file=(-1); + hid_t fid = -1; /* open the sec2 file */ - if((file=H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT))<0) + if ((fid = H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(fid); } H5E_END_TRY; - return -1; -} + + return FAIL; + +} /* end test_sec2_h5repart_opens() */ /*------------------------------------------------------------------------- @@ -132,32 +127,26 @@ error: * * Purpose: Tests h5repart-ed family files * - * Return: Success: exit(0) - * - * Failure: exit(1) - * - * Programmer: Raymond Lu - * June 1, 2005 - * - * Modifications: + * Return: EXIT_SUCCESS/EXIT_FAILURE * *------------------------------------------------------------------------- */ int main(void) { - int nerrors=0; + int nerrors = 0; - nerrors += test_family_h5repart_opens()<0 ?1:0; - nerrors += test_sec2_h5repart_opens()<0 ?1:0; + nerrors += test_family_h5repart_opens() < 0 ? 1 : 0; + nerrors += test_sec2_h5repart_opens() < 0 ? 1 : 0; - if (nerrors) goto error; + if (nerrors) + goto error; - return 0; + HDexit(EXIT_SUCCESS); error: nerrors = MAX(1, nerrors); - printf("***** %d FAMILY FILE TEST%s FAILED! *****\n", + HDprintf("***** %d FAMILY FILE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ -- cgit v0.12 From 3a7c5f19c4254046db758e28c32f1e0f305189ee Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Mon, 24 Jul 2017 16:33:48 -0500 Subject: Correct case of var --- config/cmake_ext_mod/HDFLibMacros.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/cmake_ext_mod/HDFLibMacros.cmake b/config/cmake_ext_mod/HDFLibMacros.cmake index f2e03d7..54e408b 100644 --- a/config/cmake_ext_mod/HDFLibMacros.cmake +++ b/config/cmake_ext_mod/HDFLibMacros.cmake @@ -74,14 +74,14 @@ macro (EXTERNAL_JPEG_LIBRARY compress_type jpeg_pic) HDF_IMPORT_SET_LIB_OPTIONS (jpeg-static "jpeg" STATIC "") add_dependencies (JPEG jpeg-static) set (JPEG_STATIC_LIBRARY "jpeg-static") - set (JPEG_LIBRARIES ${JPEG_static_LIBRARY}) + set (JPEG_LIBRARIES ${JPEG_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target jpeg-shared add_library(jpeg-shared SHARED IMPORTED) HDF_IMPORT_SET_LIB_OPTIONS (jpeg-shared "jpeg" SHARED "") add_dependencies (JPEG jpeg-shared) set (JPEG_SHARED_LIBRARY "jpeg-shared") - set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_shared_LIBRARY}) + set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_SHARED_LIBRARY}) endif () set (JPEG_INCLUDE_DIR_GEN "${BINARY_DIR}") @@ -167,14 +167,14 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) HDF_IMPORT_SET_LIB_OPTIONS (szip-static "szip" STATIC "") add_dependencies (SZIP szip-static) set (SZIP_STATIC_LIBRARY "szip-static") - set (SZIP_LIBRARIES ${SZIP_static_LIBRARY}) + set (SZIP_LIBRARIES ${SZIP_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target szip-shared add_library(szip-shared SHARED IMPORTED) HDF_IMPORT_SET_LIB_OPTIONS (szip-shared "szip" SHARED "") add_dependencies (SZIP szip-shared) set (SZIP_SHARED_LIBRARY "szip-shared") - set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_shared_LIBRARY}) + set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_SHARED_LIBRARY}) endif () set (SZIP_INCLUDE_DIR_GEN "${BINARY_DIR}") @@ -262,7 +262,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) HDF_IMPORT_SET_LIB_OPTIONS (zlib-static ${ZLIB_LIB_NAME} STATIC "") add_dependencies (ZLIB zlib-static) set (ZLIB_STATIC_LIBRARY "zlib-static") - set (ZLIB_LIBRARIES ${ZLIB_static_LIBRARY}) + set (ZLIB_LIBRARIES ${ZLIB_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target zlib-shared add_library(zlib-shared SHARED IMPORTED) -- cgit v0.12 From e790a0b42d0552081b6138552ef3955760894e5b Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 25 Jul 2017 15:15:20 -0500 Subject: HDFFV-10246 Add check for string not null before use --- MANIFEST | 1 + tools/lib/h5diff_array.c | 20 ++++++++++++++++---- tools/test/h5diff/CMakeTests.cmake | 8 ++++++++ tools/test/h5diff/testfiles/h5diff_vlstr.txt | 16 ++++++++++++++++ tools/test/h5diff/testh5diff.sh.in | 5 +++++ 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tools/test/h5diff/testfiles/h5diff_vlstr.txt diff --git a/MANIFEST b/MANIFEST index 27f38be..e5a038d 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2398,6 +2398,7 @@ ./tools/test/h5diff/testfiles/h5diff_udfail.txt ./tools/test/h5diff/testfiles/diff_strings1.h5 ./tools/test/h5diff/testfiles/diff_strings2.h5 +./tools/test/h5diff/testfiles/h5diff_vlstr.txt #vds ./tools/test/h5diff/testfiles/h5diff_v1.txt ./tools/test/h5diff/testfiles/h5diff_v2.txt diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index 0b1a96e..7236ad1 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -673,21 +673,33 @@ static hsize_t diff_datum(void *_mem1, h5difftrace("diff_datum H5T_STRING variable\n"); /* Get pointer to first string */ s1 = *(char**) mem1; - size1 = HDstrlen(s1); + if(s1) + size1 = HDstrlen(s1); + else + size1 = 0; /* Get pointer to second string */ s2 = *(char**) mem2; - size2 = HDstrlen(s2); + if(s2) + size2 = HDstrlen(s2); + else + size2 = 0; } else if (H5T_STR_NULLTERM == pad) { h5difftrace("diff_datum H5T_STRING null term\n"); /* Get pointer to first string */ s1 = (char*) mem1; - size1 = HDstrlen(s1); + if(s1) + size1 = HDstrlen(s1); + else + size1 = 0; if (size1 > size_mtype) size1 = size_mtype; /* Get pointer to second string */ s2 = (char*) mem2; - size2 = HDstrlen(s2); + if(s2) + size2 = HDstrlen(s2); + else + size2 = 0; if (size2 > size_mtype) size2 = size_mtype; } diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 72dda6b..02db446 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -95,6 +95,8 @@ ${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5 + # tools/testfiles + ${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5 ) set (LIST_OTHER_TEST_FILES @@ -286,6 +288,7 @@ ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v1.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v2.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v3.txt + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_vlstr.txt ) set (LIST_WIN_TEST_FILES @@ -311,6 +314,8 @@ HDFTEST_COPY_FILE("${h5_tstfiles}" "${PROJECT_BINARY_DIR}/PAR/testfiles/${fname}" "h5diff_files") endif () endforeach () + # copy second version of tvlstr.h5 + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/testfiles/tvlstr2.h5" "h5diff_files") # @@ -888,6 +893,8 @@ h5diff_v2.out.err h5diff_v3.out h5diff_v3.out.err + h5diff_vlstr.out + h5diff_vlstr.out.err ) set_tests_properties (H5DIFF-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") if (NOT "${last_test}" STREQUAL "") @@ -1465,6 +1472,7 @@ ADD_H5_TEST (h5diff_487 1 -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 # # diff various multiple vlen and fixed strings in a compound type dataset # ############################################################################## ADD_H5_TEST (h5diff_530 0 -v ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy) +ADD_H5_TEST (h5diff_vlstr 0 -v tvlstr.h5 tvlstr2.h5) # ############################################################################## # # Test container types (array,vlen) with multiple nested compound types diff --git a/tools/test/h5diff/testfiles/h5diff_vlstr.txt b/tools/test/h5diff/testfiles/h5diff_vlstr.txt new file mode 100644 index 0000000..67141f1 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_vlstr.txt @@ -0,0 +1,16 @@ + +file1 file2 +--------------------------------------- + x x / + x x /Dataset1 + x x /vl_string_type + +group : and +0 differences found +attribute: > and > +0 differences found +dataset: and +0 differences found +datatype: and +0 differences found +EXIT CODE: 0 diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index d769c23..3450d30 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -124,6 +124,7 @@ $SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.reference.h5 $SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.output.h5 $SRC_H5DIFF_TESTFILES/diff_strings1.h5 $SRC_H5DIFF_TESTFILES/diff_strings2.h5 +$SRC_TOOLS_TESTFILES/tvlstr.h5 " LIST_HDF5_VDS_TEST_FILES=" @@ -341,6 +342,7 @@ $SRC_H5DIFF_TESTFILES/h5diff_tmp2.txt $SRC_H5DIFF_TESTFILES/h5diff_v1.txt $SRC_H5DIFF_TESTFILES/h5diff_v2.txt $SRC_H5DIFF_TESTFILES/h5diff_v3.txt +$SRC_H5DIFF_TESTFILES/h5diff_vlstr.txt " # @@ -566,6 +568,8 @@ SKIP() { ############################################################################## # prepare for test COPY_TESTFILES_TO_TESTDIR +# second copy of tvlstr.h5 +$CP -f $SRC_TOOLS_TESTFILES/tvlstr.h5 $TESTDIR/tvlstr.h5 # ############################################################################ # # Common usage @@ -1141,6 +1145,7 @@ TOOLTEST h5diff_487.txt -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 h5 # # diff various multiple vlen and fixed strings in a compound type dataset # ############################################################################## TOOLTEST h5diff_530.txt -v h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group /group_copy +TOOLTEST h5diff_vlstr.txt -v tvlstr.h5 tvlstr2.h5 # ############################################################################## # # Test container types (array,vlen) with multiple nested compound types -- cgit v0.12 From 1018256d55516c376204d8f92fcceb0c5e518aba Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 25 Jul 2017 15:44:25 -0500 Subject: Correct copy to file name --- tools/test/h5diff/testh5diff.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index 3450d30..df472d4 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -569,7 +569,7 @@ SKIP() { # prepare for test COPY_TESTFILES_TO_TESTDIR # second copy of tvlstr.h5 -$CP -f $SRC_TOOLS_TESTFILES/tvlstr.h5 $TESTDIR/tvlstr.h5 +$CP -f $SRC_TOOLS_TESTFILES/tvlstr.h5 $TESTDIR/tvlstr2.h5 # ############################################################################ # # Common usage -- cgit v0.12 From 08d63d17f9ba80f205d54bc6526cb709a81ff0cb Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 25 Jul 2017 16:58:44 -0500 Subject: Add the release note --- release_docs/RELEASE.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 7eb70e1..cd29900 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -147,6 +147,15 @@ Bug Fixes since HDF5-1.10.1 release Tools ----- + - h5diff + + h5diff segfaulted on compare of a NULL variable length string. + + Improved h5diff compare of strings by adding a check for + NULL strings and setting the lengths to zero. + + (ADB - 2017/07/25, HDFFV-10246) + - h5import h5import crashed trying to import data from a subset of a dataset. -- cgit v0.12 From 373a37da01564a4cb33662ff52d176bf9d5a3b75 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 26 Jul 2017 09:30:27 -0500 Subject: Make sure zip_perf program does not run concurrently --- tools/test/perform/CMakeTests.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/test/perform/CMakeTests.cmake b/tools/test/perform/CMakeTests.cmake index 2933563..39faa73 100644 --- a/tools/test/perform/CMakeTests.cmake +++ b/tools/test/perform/CMakeTests.cmake @@ -146,6 +146,7 @@ else () -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" ) endif () +set_tests_properties (PERFORM_zip_perf PROPERTIES DEPENDS PERFORM_zip_perf_help) if (H5_HAVE_PARALLEL) add_test (NAME PERFORM_h5perf COMMAND ${MPIEXEC} ${MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_POSTFLAGS} $) -- cgit v0.12 From cd0d804611f63a1412a8c8d2a3cc0e7fd047a146 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 26 Jul 2017 09:51:57 -0500 Subject: Verify default paths exist in table --- test/plugin.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/plugin.c b/test/plugin.c index 1254e9a..573b72c 100644 --- a/test/plugin.c +++ b/test/plugin.c @@ -734,6 +734,8 @@ test_filter_path_apis(void) if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) TEST_ERROR H5PLsize(&ndx); + if(ndx!=2) TEST_ERROR + PASSED(); TESTING(" remove"); /* Remove all existing paths*/ -- cgit v0.12 From 55d82c8f747b3a817a4a0b6b12e506b9c2dc0878 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 26 Jul 2017 10:45:08 -0500 Subject: Add test header for log --- test/plugin.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/plugin.c b/test/plugin.c index 573b72c..3034c0b 100644 --- a/test/plugin.c +++ b/test/plugin.c @@ -733,6 +733,7 @@ test_filter_path_apis(void) if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) TEST_ERROR + TESTING(" initialize"); H5PLsize(&ndx); if(ndx!=2) TEST_ERROR PASSED(); -- cgit v0.12 From b7f19967d76f50890ff6ef1a7a756dc37c555538 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 27 Jul 2017 08:52:21 -0500 Subject: Removed FCFLAGS when building the examples since they should already be included the h5fc wrapper. NAG complains about doubly declared options. --- fortran/examples/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fortran/examples/Makefile.am b/fortran/examples/Makefile.am index db85c09..6bf2edb 100644 --- a/fortran/examples/Makefile.am +++ b/fortran/examples/Makefile.am @@ -66,10 +66,10 @@ h5_crtatt.chkexe_: h5_rdwt.chkexe_ # Additional dependencies for the examples are listed below if BUILD_PARALLEL_CONDITIONAL $(EXTRA_PROG): $(H5FC_PP) - $(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; else $(EXTRA_PROG): $(H5FC) - $(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; endif # Tell automake how to install examples -- cgit v0.12 From 04d6a3ec3c91cb30dce5dd5abb1b67021527c2fb Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Thu, 27 Jul 2017 11:18:52 -0500 Subject: if PARALLEL copy seconf file to PAR folder --- tools/test/h5diff/CMakeTests.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 02db446..608795c 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -316,6 +316,9 @@ endforeach () # copy second version of tvlstr.h5 HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/testfiles/tvlstr2.h5" "h5diff_files") + if (H5_HAVE_PARALLEL) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/PAR/testfiles/tvlstr2.h5" "h5diff_files") + endif () # -- cgit v0.12 From be5257f8766e5be1ceacee3260b4b52b499983f6 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 27 Jul 2017 11:32:21 -0500 Subject: libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler on Linux (other OSs have not been tested). Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) --- configure.ac | 44 ++++++++++++++++++++++++++++++++------------ fortran/src/h5fc.in | 2 +- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 163f6d3..fe999ce 100644 --- a/configure.ac +++ b/configure.ac @@ -3338,6 +3338,38 @@ else STATIC_SHARED="none" fi +if test "X$HDF_FORTRAN" = "Xyes"; then + chmod 755 fortran/src/h5fc + ## libtool does not pass the correct argument linker (wl=) for the Intel Fortran compiler + ## on OS X, which is needed when building shared libraries on OS X. This script + ## replaces the 3rd occurrence, which is for Fortran, of wl="" with wl="-Wl," (HDFFV-2772) + case "`uname`" in + Darwin*) + cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool + ;; + esac + + ### libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler + ### on Linux (other OSs have not been tested). + ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to + ### wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) + case "`uname`" in + Linux*) + + fortran_linux_linker_option="-Wl," + + if (grep -i 'NAG_Fortran' libtool > /dev/null); then + cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool + fortran_linux_linker_option="-Wl,-Wl,," + fi + + ## Set the correct linker option for use in h5fc.in markup + AC_SUBST([fortran_linux_linker_option]) + ;; + esac + +fi + ## ---------------------------------------------------------------------- ## Set a macro if shared library is enabled. ## @@ -3474,18 +3506,6 @@ AC_OUTPUT chmod 755 tools/src/misc/h5cc -if test "X$HDF_FORTRAN" = "Xyes"; then - chmod 755 fortran/src/h5fc - ## libtool does not pass the correct argument linker (wl=) for the Intel Fortran compiler - ## on OS X, which is needed when building shared libraries on OS X. This script - ## replaces the 3rd occurrence, which is for Fortran, of wl="" with wl="-Wl," (HDFFV-2772) - case "`uname`" in - Darwin*) - cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool - ;; - esac -fi - if test "X$HDF_CXX" = "Xyes"; then chmod 755 c++/src/h5c++ fi diff --git a/fortran/src/h5fc.in b/fortran/src/h5fc.in index 47642c9..29ef83f 100644 --- a/fortran/src/h5fc.in +++ b/fortran/src/h5fc.in @@ -307,7 +307,7 @@ if test "x$do_link" = "xyes"; then link_args="$link_args -L${libdir}" case "$host_os" in - linux*) flag="-Wl,-rpath -Wl," ;; + linux*) flag="@fortran_linux_linker_option@-rpath -Wl," ;; hpux*) flag="-Wl,+b -Wl," ;; freebsd*|solaris*) flag="-R" ;; rs6000*|aix*) flag="-L" ;; -- cgit v0.12 From d138f164dddb2fd69b7325a861037f4b54406a71 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 27 Jul 2017 12:57:48 -0500 Subject: removed chmod 755 for h5fc --- configure.ac | 1 - 1 file changed, 1 deletion(-) diff --git a/configure.ac b/configure.ac index fe999ce..1ca7a34 100644 --- a/configure.ac +++ b/configure.ac @@ -3339,7 +3339,6 @@ else fi if test "X$HDF_FORTRAN" = "Xyes"; then - chmod 755 fortran/src/h5fc ## libtool does not pass the correct argument linker (wl=) for the Intel Fortran compiler ## on OS X, which is needed when building shared libraries on OS X. This script ## replaces the 3rd occurrence, which is for Fortran, of wl="" with wl="-Wl," (HDFFV-2772) -- cgit v0.12 From 92491aa4b6dc2d4fa3c394e0aa928300379e1ced Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 27 Jul 2017 13:09:36 -0500 Subject: added chmod 755 for h5fc --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 1ca7a34..45bd4d1 100644 --- a/configure.ac +++ b/configure.ac @@ -3504,7 +3504,9 @@ AC_CONFIG_COMMANDS([.classes], [], [$MKDIR_P java/src/.classes; AC_OUTPUT chmod 755 tools/src/misc/h5cc - +if test "X$HDF_FORTRAN" = "Xyes"; then + chmod 755 fortran/src/h5fc +fi if test "X$HDF_CXX" = "Xyes"; then chmod 755 c++/src/h5c++ fi -- cgit v0.12 From c08ee778588c9e0752e242c201000f1f11a0b742 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Fri, 28 Jul 2017 09:05:39 -0500 Subject: Rearranged where the substitution occurs. libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler on Linux (other OSs have not been tested). Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) --- configure.ac | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index 45bd4d1..47f5897 100644 --- a/configure.ac +++ b/configure.ac @@ -3339,14 +3339,6 @@ else fi if test "X$HDF_FORTRAN" = "Xyes"; then - ## libtool does not pass the correct argument linker (wl=) for the Intel Fortran compiler - ## on OS X, which is needed when building shared libraries on OS X. This script - ## replaces the 3rd occurrence, which is for Fortran, of wl="" with wl="-Wl," (HDFFV-2772) - case "`uname`" in - Darwin*) - cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool - ;; - esac ### libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler ### on Linux (other OSs have not been tested). @@ -3358,7 +3350,6 @@ if test "X$HDF_FORTRAN" = "Xyes"; then fortran_linux_linker_option="-Wl," if (grep -i 'NAG_Fortran' libtool > /dev/null); then - cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool fortran_linux_linker_option="-Wl,-Wl,," fi @@ -3504,13 +3495,35 @@ AC_CONFIG_COMMANDS([.classes], [], [$MKDIR_P java/src/.classes; AC_OUTPUT chmod 755 tools/src/misc/h5cc -if test "X$HDF_FORTRAN" = "Xyes"; then - chmod 755 fortran/src/h5fc -fi if test "X$HDF_CXX" = "Xyes"; then chmod 755 c++/src/h5c++ fi + +if test "X$HDF_FORTRAN" = "Xyes"; then + chmod 755 fortran/src/h5fc + ## libtool does not pass the correct argument linker (wl=) for the Intel Fortran compiler + ## on OS X, which is needed when building shared libraries on OS X. This script + ## replaces the 3rd occurrence, which is for Fortran, of wl="" with wl="-Wl," (HDFFV-2772) + case "`uname`" in + Darwin*) + cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool + ;; + esac + + ### libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler + ### on Linux (other OSs have not been tested). + ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to + ### wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) + case "`uname`" in + Linux*) + if (grep -i 'NAG_Fortran' libtool > /dev/null); then + cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool + fi + ;; + esac +fi + ## HDF5 configure code created by autotools with gcc 4.9.2 is adding problematic ## linker flags: -l with no library name; -l , specifically gfortran or m. ## This sed script corrects "-l " first and then "-l " with no library name. -- cgit v0.12 From 59e94f5009a1f738d9924be942448f8cf6355b00 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Sat, 29 Jul 2017 01:10:25 -0500 Subject: Add h5repack tests for paged aggregation Add tests to h5repack.sh.in to verify options added for paged aggregation work as expected. --- MANIFEST | 12 +++ tools/test/h5repack/h5repack.sh.in | 103 +++++++++++++++++++++ .../testfiles/GS.h5repack_paged_nopersist.h5.ddl | 15 +++ .../testfiles/S.h5repack_fsm_aggr_persist.h5.ddl | 15 +++ .../SP.h5repack_fsm_aggr_nopersist.h5.ddl | 16 ++++ .../testfiles/SP.h5repack_paged_persist.h5.ddl | 15 +++ .../h5repack/testfiles/SPT.h5repack_aggr.h5.ddl | 16 ++++ .../h5repack/testfiles/STG.h5repack_none.h5.ddl | 15 +++ tools/test/h5repack/testfiles/h5repack_aggr.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_nopersist.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_persist.h5 | Bin 0 -> 2565 bytes tools/test/h5repack/testfiles/h5repack_none.h5 | Bin 0 -> 1808 bytes .../h5repack/testfiles/h5repack_paged_nopersist.h5 | Bin 0 -> 8192 bytes .../h5repack/testfiles/h5repack_paged_persist.h5 | Bin 0 -> 16384 bytes 14 files changed, 207 insertions(+) create mode 100644 tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl create mode 100644 tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl create mode 100644 tools/test/h5repack/testfiles/h5repack_aggr.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_none.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_persist.h5 diff --git a/MANIFEST b/MANIFEST index 27f38be..af4dc7e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2405,6 +2405,7 @@ #test files for h5repack ./tools/test/h5repack/testfiles/README +./tools/test/h5repack/testfiles/h5repack_aggr.h5 ./tools/test/h5repack/testfiles/h5repack_attr.h5 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 ./tools/test/h5repack/testfiles/h5repack_deflate.h5 @@ -2414,6 +2415,8 @@ ./tools/test/h5repack/testfiles/h5repack_fill.h5 ./tools/test/h5repack/testfiles/h5repack_filters.h5 ./tools/test/h5repack/testfiles/h5repack_fletcher.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 ./tools/test/h5repack/testfiles/h5repack_hlink.h5 ./tools/test/h5repack/testfiles/h5repack.info ./tools/test/h5repack/testfiles/h5repack_layout.h5 @@ -2425,7 +2428,10 @@ ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5 ./tools/test/h5repack/testfiles/h5repack_nbit.h5 +./tools/test/h5repack/testfiles/h5repack_none.h5 ./tools/test/h5repack/testfiles/h5repack_objs.h5 +./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_paged_persist.h5 ./tools/test/h5repack/testfiles/h5repack_refs.h5 ./tools/test/h5repack/testfiles/h5repack_shuffle.h5 ./tools/test/h5repack/testfiles/h5repack_soffset.h5 @@ -2444,6 +2450,12 @@ ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst +./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl +./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl +./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl +./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl ./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl ./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl ./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index bc6b527..e474bc3 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../../src/h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary +H5STAT=../../src/h5stat/h5stat # The h5stat tool name +H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary + RM='rm -rf' CMP='cmp' DIFF='diff -c' @@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_none.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 @@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl $SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl +$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl " # @@ -707,6 +722,58 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is similar to TOOLTEST_DUMP(). +# Test h5repack with options added for paged aggregation. +# h5stat is used on the repacked file and the expected output +# is compared for correctness. +# +TOOLTEST_STAT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5stat output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5STAT_BIN -S -s $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # TOOLTEST_META: # Test metadata block size option. # Reason to create a function here is to localize all special steps related to @@ -1057,6 +1124,42 @@ fi arg="tordergr.h5 -L" TOOLTEST_DUMP crtorder $arg +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# +arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_fsm_aggr_persist.h5 -S AGGR" +TOOLTEST_STAT S $arg +# +arg="h5repack_none.h5 -S PAGE -T 10 -G 2048" +TOOLTEST_STAT STG $arg +# +arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR" +TOOLTEST_STAT GS $arg +# +arg="h5repack_paged_persist.h5 -S NONE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5" +TOOLTEST_STAT SPT $arg + + ######################################################### # layout options (these files have no filters) ######################################################### diff --git a/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl new file mode 100644 index 0000000..d8ca992 --- /dev/null +++ b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-GS.h5repack_paged_nopersist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 512 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl new file mode 100644 index 0000000..87ccbc4 --- /dev/null +++ b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-S.h5repack_fsm_aggr_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl new file mode 100644 index 0000000..affa93b --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SP.h5repack_fsm_aggr_nopersist.h5 +Free-space persist: TRUE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 1000 - 9999: 2 + Total # of sections: 2 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1602 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 6307 bytes/38.5% + Unaccounted space: 8075 bytes +Total space: 16384 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl new file mode 100644 index 0000000..1084090 --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-SP.h5repack_paged_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_NONE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 0 bytes +Total space: 1672 bytes diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl new file mode 100644 index 0000000..1ce06b2 --- /dev/null +++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SPT.h5repack_aggr.h5 +Free-space persist: TRUE +Free-space section threshold: 5 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 100 - 999: 1 + Total # of sections: 1 +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1485 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 680 bytes/26.5% + Unaccounted space: 0 bytes +Total space: 2565 bytes diff --git a/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl new file mode 100644 index 0000000..51cd7e5 --- /dev/null +++ b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-STG.h5repack_none.h5 +Free-space persist: FALSE +Free-space section threshold: 10 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 2048 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 2424 bytes +Total space: 4096 bytes diff --git a/tools/test/h5repack/testfiles/h5repack_aggr.h5 b/tools/test/h5repack/testfiles/h5repack_aggr.h5 new file mode 100644 index 0000000..8da0751 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_aggr.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 new file mode 100644 index 0000000..1cd5f3f Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 new file mode 100644 index 0000000..469dbce Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_none.h5 b/tools/test/h5repack/testfiles/h5repack_none.h5 new file mode 100644 index 0000000..2e4d789 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_none.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 new file mode 100644 index 0000000..352e586 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 new file mode 100644 index 0000000..64c3b23 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 differ -- cgit v0.12 From 9bcf8b2f2568083449ae3f9b6c2efbf6ed7f413a Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 1 Aug 2017 04:59:36 -0700 Subject: Minor tweaks in response to code review. --- src/H5system.c | 8 ++------ test/plugin.c | 56 +++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/H5system.c b/src/H5system.c index 1e718e7..a1cdf19 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1266,15 +1266,11 @@ H5_expand_windows_env_vars(char **env_var) HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") /* Expand the environment variable string */ - if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE) { - temp_buf = (char *)H5MM_xfree(temp_buf); + if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE) HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") - } - if (n_chars == 0) { - temp_buf = (char *)H5MM_xfree(temp_buf); + if (0 == n_chars) HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") - } *env_var = (char *)H5MM_xfree(*env_var); *env_var = temp_buf; diff --git a/test/plugin.c b/test/plugin.c index bcd7348..ea199f6 100644 --- a/test/plugin.c +++ b/test/plugin.c @@ -723,7 +723,6 @@ test_path_api_calls(void) ssize_t path_len = -1; char path[256]; char temp_name[256]; - herr_t ret_value = -1; HDputs("Testing access to the filter path table"); @@ -1106,37 +1105,44 @@ main(void) /* Testing setup */ h5_reset(); - fapl = h5_fileaccess(); + + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Turn off the chunk cache, so all the chunks are immediately written to disk */ - if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) TEST_ERROR + if (H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) + TEST_ERROR rdcc_nbytes = 0; - if(H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) TEST_ERROR + if (H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) + TEST_ERROR /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR + if ((fapl2 = H5Pcopy(fapl)) < 0) + TEST_ERROR /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + if (H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR - h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { + for (new_format = FALSE; new_format <= TRUE; new_format++) { hid_t my_fapl; /* Set the FAPL for the type of format */ - if(new_format) { + if (new_format) { HDputs("\nTesting with new file format:"); my_fapl = fapl2; - } /* end if */ + } else { HDputs("Testing with old file format:"); my_fapl = fapl; - } /* end else */ + } /* Create the file for this test */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) + TEST_ERROR /* Test dynamically loaded filters for chunked dataset */ nerrors += (test_filters_for_datasets(file) < 0 ? 1 : 0); @@ -1144,12 +1150,15 @@ main(void) /* Test dynamically loaded filters for groups */ nerrors += (test_filters_for_groups(file) < 0 ? 1 : 0); - if(H5Fclose(file) < 0) TEST_ERROR + if (H5Fclose(file) < 0) + TEST_ERROR } /* end for */ /* Close FAPL */ - if(H5Pclose(fapl2) < 0) TEST_ERROR - if(H5Pclose(fapl) < 0) TEST_ERROR + if (H5Pclose(fapl2) < 0) + TEST_ERROR + if (H5Pclose(fapl) < 0) + TEST_ERROR /* Restore the default error handler (set in h5_reset()) */ h5_restore_err(); @@ -1158,10 +1167,12 @@ main(void) /* Close the library so that all loaded plugin libraries are unloaded */ h5_reset(); - fapl = h5_fileaccess(); + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Reopen the file for testing data reading */ - if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR + if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + TEST_ERROR /* Read the data with filters */ nerrors += (test_read_with_filters(file) < 0 ? 1 : 0); @@ -1174,20 +1185,23 @@ main(void) /* Close the library so that all loaded plugin libraries are unloaded */ h5_reset(); - fapl = h5_fileaccess(); + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Reopen the file for testing data reading */ - if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR + if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + TEST_ERROR /* Read the data with disabled filters */ nerrors += (test_noread_with_filters(file) < 0 ? 1 : 0); - if(H5Fclose(file) < 0) TEST_ERROR + if (H5Fclose(file) < 0) + TEST_ERROR /* Test the APIs for access to the filter plugin path table */ nerrors += (test_path_api_calls() < 0 ? 1 : 0); - if(nerrors) + if (nerrors) TEST_ERROR HDprintf("All plugin tests passed.\n"); -- cgit v0.12 From cf2da9a3d3d608c8763be06e3f036e3d6cb12e72 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Tue, 1 Aug 2017 08:46:23 -0500 Subject: Removed extra Fortran flags, (HDFFV-10037) --- hl/fortran/examples/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hl/fortran/examples/Makefile.am b/hl/fortran/examples/Makefile.am index 6709fb7..d383f9a 100644 --- a/hl/fortran/examples/Makefile.am +++ b/hl/fortran/examples/Makefile.am @@ -43,10 +43,10 @@ FORTRAN_API=yes # Tell automake how to build examples using h5fc if BUILD_PARALLEL_CONDITIONAL $(EXTRA_PROG): $(H5FC_PP) - $(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; else $(EXTRA_PROG): $(H5FC) - $(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; endif # Tell automake how to install examples -- cgit v0.12 From 538e9103681290b6a30d14fc94f50e499b2d11d3 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 1 Aug 2017 13:36:32 -0500 Subject: HDFFV-10256 correct len of string copy to the len of esc string --- tools/src/h5dump/h5dump_xml.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index bb0fd7a..49d4f96 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -780,20 +780,20 @@ xml_escape_the_string(const char *str, int slen) esc_len = 1; } else if (*cp == '\'') { - HDstrncpy(ncp, apos, ncp_len); esc_len = HDstrlen(apos); + HDstrncpy(ncp, apos, esc_len); } else if (*cp == '<') { - HDstrncpy(ncp, lt, ncp_len); esc_len = HDstrlen(lt); + HDstrncpy(ncp, lt, esc_len); } else if (*cp == '>') { - HDstrncpy(ncp, gt, ncp_len); esc_len = HDstrlen(gt); + HDstrncpy(ncp, gt, esc_len); } else if (*cp == '&') { - HDstrncpy(ncp, amp, ncp_len); esc_len = HDstrlen(amp); + HDstrncpy(ncp, amp, esc_len); } else { *ncp = *cp; -- cgit v0.12 From e1a81b17e027a1f666b53131d6dd1782a3b9704b Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 1 Aug 2017 13:52:10 -0500 Subject: HDFFV-10256 Add test --- MANIFEST | 2 ++ tools/test/h5dump/CMakeTestsXML.cmake | 7 +++++++ tools/test/h5dump/testh5dumpxml.sh.in | 5 +++++ tools/testfiles/test35.nc | Bin 0 -> 14297 bytes tools/testfiles/test35.nc.xml | 20 ++++++++++++++++++++ 5 files changed, 34 insertions(+) create mode 100644 tools/testfiles/test35.nc create mode 100644 tools/testfiles/test35.nc.xml diff --git a/MANIFEST b/MANIFEST index c7729e8..2fc5678 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2092,6 +2092,8 @@ ./tools/testfiles/tdset2.h5.xml ./tools/testfiles/tempty.h5.xml ./tools/testfiles/tenum.h5.xml +./tools/testfiles/test35.nc +./tools/testfiles/test35.nc.xml ./tools/testfiles/tfpformat.h5.xml ./tools/testfiles/tgroup.h5.xml ./tools/testfiles/thlink.h5.xml diff --git a/tools/test/h5dump/CMakeTestsXML.cmake b/tools/test/h5dump/CMakeTestsXML.cmake index 92e08a0..bdc6755 100644 --- a/tools/test/h5dump/CMakeTestsXML.cmake +++ b/tools/test/h5dump/CMakeTestsXML.cmake @@ -50,6 +50,7 @@ ${HDF5_TOOLS_DIR}/testfiles/tname-quot.h5 ${HDF5_TOOLS_DIR}/testfiles/tname-sp.h5 ${HDF5_TOOLS_DIR}/testfiles/tnamed_dtype_attr.h5 + ${HDF5_TOOLS_DIR}/testfiles/test35.nc ${HDF5_TOOLS_DIR}/testfiles/tnestedcomp.h5 ${HDF5_TOOLS_DIR}/testfiles/tnodata.h5 ${HDF5_TOOLS_DIR}/testfiles/tobjref.h5 @@ -99,6 +100,7 @@ ${HDF5_TOOLS_DIR}/testfiles/tempty-ns.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tempty-ns-2.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tenum.h5.xml + ${HDF5_TOOLS_DIR}/testfiles/test35.nc.xml ${HDF5_TOOLS_DIR}/testfiles/textlink.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tfpformat.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tgroup.h5.xml @@ -258,6 +260,8 @@ tempty.h5.out.err tenum.h5.out tenum.h5.out.err + test35.nc.out + test35.nc.out.err textlink.h5.out textlink.h5.out.err tfpformat.h5.out @@ -430,3 +434,6 @@ # tests for floating point user defined printf format ADD_XML_H5_TEST (tfpformat.h5 0 -u -m %.7f tfpformat.h5) + # test for HDFFV-10256 issue + ADD_XML_H5_TEST (test35.nc 0 test35.nc) + diff --git a/tools/test/h5dump/testh5dumpxml.sh.in b/tools/test/h5dump/testh5dumpxml.sh.in index 5f62946..39b1361 100644 --- a/tools/test/h5dump/testh5dumpxml.sh.in +++ b/tools/test/h5dump/testh5dumpxml.sh.in @@ -80,6 +80,7 @@ $SRC_H5DUMP_TESTFILES/tdset.h5 $SRC_H5DUMP_TESTFILES/tdset2.h5 $SRC_H5DUMP_TESTFILES/tempty.h5 $SRC_H5DUMP_TESTFILES/tenum.h5 +$SRC_H5DUMP_TESTFILES/test35.nc $SRC_H5DUMP_TESTFILES/textlink.h5 $SRC_H5DUMP_TESTFILES/tfpformat.h5 $SRC_H5DUMP_TESTFILES/tgroup.h5 @@ -144,6 +145,7 @@ $SRC_H5DUMP_TESTFILES/tempty-nons-uri.h5.xml $SRC_H5DUMP_TESTFILES/tempty-ns.h5.xml $SRC_H5DUMP_TESTFILES/tempty-ns-2.h5.xml $SRC_H5DUMP_TESTFILES/tenum.h5.xml +$SRC_H5DUMP_TESTFILES/test35.nc.xml $SRC_H5DUMP_TESTFILES/textlink.h5.xml $SRC_H5DUMP_TESTFILES/tfpformat.h5.xml $SRC_H5DUMP_TESTFILES/tgroup.h5.xml @@ -383,6 +385,9 @@ TOOLTEST torderattr4.h5.xml --xml -H --sort_by=creation_order --sort_order=desce # tests for floating point user defined printf format TOOLTEST tfpformat.h5.xml -u -m %.7f tfpformat.h5 +# test for HDFFV-10256 issue +TOOLTEST test35.nc --xml test35.nc + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/testfiles/test35.nc b/tools/testfiles/test35.nc new file mode 100644 index 0000000..4bd5d7f Binary files /dev/null and b/tools/testfiles/test35.nc differ diff --git a/tools/testfiles/test35.nc.xml b/tools/testfiles/test35.nc.xml new file mode 100644 index 0000000..a38ed8a --- /dev/null +++ b/tools/testfiles/test35.nc.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>a<aaaaaa>aaaaaaaa<aaaaaa>a<aaaaaa>aaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaa<aaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"aaaaa\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaa<aaaa>aaaa<aaaa>>aaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<a>aaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<>aaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + + + + -- cgit v0.12 From 2d932366b27af21b9197c882b4bf09f7c7bc973e Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Tue, 1 Aug 2017 15:33:43 -0500 Subject: added test for NAG compiler and use the FC_BASENAME for NAG detection, (HDFFV-10037) --- config/linux-gnulibc1 | 65 ++++++++++++++++++++++++++++++++++++++++++++++++--- configure.ac | 3 +-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1 index 1785e0e..7cd1a2a 100644 --- a/config/linux-gnulibc1 +++ b/config/linux-gnulibc1 @@ -84,7 +84,20 @@ else $RM $tmpfile fc_version_info=`$FC -V | grep Absoft` ;; - + # The NAG compiler + nagfor*|nagftn*) + RM='rm -f' + tmpfile=/tmp/cmpver.$$ + $FC -V >& $tmpfile + if test -s "$tmpfile"; then + if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then + FC_BASENAME=nagfor + fi + fi + fc_version_info=`grep "NAG Fortran" $tmpfile` + echo "compiler '$FC' is $fc_version_info" + $RM $tmpfile + ;; *) ;; esac @@ -108,10 +121,10 @@ case $FC_BASENAME in # f95) # Set required flag for compiling C stubs - H5_CFLAGS="$H5_CFLAGS" + H5_CFLAGS="$H5_CFLAGS" F9XSUFFIXFLAG="" -# We force compiler to use upper case for external names +# We force compiler to use upper case for external names # (just in case since this should be a default EIP) H5_FCFLAGS="$H5_FCFLAGS" FSEARCH_DIRS="" @@ -136,6 +149,39 @@ case $FC_BASENAME in f9x_flags_set=yes ;; +# +# NAG compiler +# + nagfor) + # Set required flag for compiling C stubs + H5_CFLAGS="$H5_CFLAGS" + + F9XSUFFIXFLAG="" +# We force compiler to use upper case for external names +# (just in case since this should be a default EIP) + H5_FCFLAGS="$H5_FCFLAGS" + FSEARCH_DIRS="" + + # Production + PROD_FCFLAGS= + + # Debug + DEBUG_FCFLAGS= + + # Symbols + SYMBOLS_FCFLAGS="-g" + NO_SYMBOLS_FCFLAGS="" + + # Profiling + PROFILE_FCFLAGS="" + + # Optimization + HIGH_OPT_FCFLAGS="-O" + DEBUG_OPT_FCFLAGS= + NO_OPT_FCFLAGS= + + f9x_flags_set=yes + ;; esac @@ -230,6 +276,19 @@ case $FC in *pgf90*) fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'pgf90'` ;; + *nagfor*|*nagftn*) + RM='rm -f' + tmpfile=/tmp/cmpver.$$ + $FC -V >& $tmpfile + if test -s "$tmpfile"; then + if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then + FC_BASENAME=nagfor + fi + fi + fc_version_info=`grep "NAG Fortran" $tmpfile` + $RM $tmpfile + echo "compiler '$FC' is $fc_version_info" + ;; *) echo "No match to get fc_version_info for $FC" diff --git a/configure.ac b/configure.ac index 47f5897..cf82a2e 100644 --- a/configure.ac +++ b/configure.ac @@ -3348,8 +3348,7 @@ if test "X$HDF_FORTRAN" = "Xyes"; then Linux*) fortran_linux_linker_option="-Wl," - - if (grep -i 'NAG_Fortran' libtool > /dev/null); then + if test "X$FC_BASENAME" = "Xnagfor"; then fortran_linux_linker_option="-Wl,-Wl,," fi -- cgit v0.12 From 00d97d556b3e57129d8c7bbecf8b15497b3f056b Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Tue, 1 Aug 2017 15:44:55 -0500 Subject: added test for NAG compiler and use the FC_BASENAME for NAG detection, (HDFFV-10037) --- config/linux-gnulibc1 | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1 index 7cd1a2a..39225e7 100644 --- a/config/linux-gnulibc1 +++ b/config/linux-gnulibc1 @@ -153,8 +153,6 @@ case $FC_BASENAME in # NAG compiler # nagfor) - # Set required flag for compiling C stubs - H5_CFLAGS="$H5_CFLAGS" F9XSUFFIXFLAG="" # We force compiler to use upper case for external names @@ -166,19 +164,19 @@ case $FC_BASENAME in PROD_FCFLAGS= # Debug - DEBUG_FCFLAGS= + DEBUG_FCFLAGS="-C" # Symbols SYMBOLS_FCFLAGS="-g" - NO_SYMBOLS_FCFLAGS="" + NO_SYMBOLS_FCFLAGS="-s" # Profiling - PROFILE_FCFLAGS="" + PROFILE_FCFLAGS="-pg" # Optimization HIGH_OPT_FCFLAGS="-O" - DEBUG_OPT_FCFLAGS= - NO_OPT_FCFLAGS= + DEBUG_OPT_FCFLAGS="-O0" + NO_OPT_FCFLAGS="-O0" f9x_flags_set=yes ;; -- cgit v0.12 From f4e9bc21fa3d3d678ca80083dbd412b34c3ab91b Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 1 Aug 2017 15:45:17 -0500 Subject: HDFFV-10256 add release note --- release_docs/RELEASE.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index cd29900..2dbe7e9 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -147,6 +147,16 @@ Bug Fixes since HDF5-1.10.1 release Tools ----- + - h5dump + + h5dump segfaulted on output of XML file. + + Function that escape'd strings used the full buffer length + instead of just the length of the replacement string in a + strncpy call. Using the correct length fixed the issue. + + (ADB - 2017/08/01, HDFFV-10256) + - h5diff h5diff segfaulted on compare of a NULL variable length string. -- cgit v0.12 From 81340c609ba5049b34845ff2f06b2025ec5d5b22 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 2 Aug 2017 08:32:31 -0700 Subject: Moved H5PL diagnostic pragmas to comply with old gcc requirements that they be outside of functions. --- src/H5PLint.c | 21 ++++++++++++--------- src/H5PLplugin_cache.c | 15 +++++++-------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/H5PLint.c b/src/H5PLint.c index bd6bf7d..c887f86 100644 --- a/src/H5PLint.c +++ b/src/H5PLint.c @@ -292,6 +292,17 @@ done: * *------------------------------------------------------------------------- */ +/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting + * object pointers to function pointers, which is undefined in ANSI C. + * This is basically unavoidable due to the nature of dlsym() and *is* + * defined in POSIX, so it's fine. + * + * This pragma only needs to surround the assignment of the + * get_plugin_info function pointer, but early (4.4.7, at least) gcc + * only allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" herr_t H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info) { @@ -318,19 +329,10 @@ H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const v HGOTO_DONE(SUCCEED); } - /* Return a handle for the function H5PLget_plugin_info in the dynamic library. * The plugin library is suppose to define this function. - * - * NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting - * object pointers to function pointers, which is undefined in ANSI C. - * This is basically unavoidable due to the nature of dlsym() and *is* - * defined in POSIX, so it's fine. */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { -#pragma GCC diagnostic pop const H5Z_class2_t *info; @@ -358,6 +360,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5PL__open() */ +#pragma GCC diagnostic pop /*------------------------------------------------------------------------- diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c index 30a0798..d826ba0 100644 --- a/src/H5PLplugin_cache.c +++ b/src/H5PLplugin_cache.c @@ -250,6 +250,11 @@ done: * *------------------------------------------------------------------------- */ +/* See the other use of H5PL_GET_LIB_FUNC() for an explanation + * for why we disable -Wpedantic here. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) { @@ -276,16 +281,9 @@ H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *f H5PL_get_plugin_info_t get_plugin_info_function; const H5Z_class2_t *filter_info; - /* Get the "get plugin info" function from the plugin. - * - * See the other use of H5PL_GET_LIB_FUNC() for an explanation - * for why we disable -Wpedantic here. - */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" + /* Get the "get plugin info" function from the plugin. */ if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info"))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") -#pragma GCC diagnostic pop /* Call the "get plugin info" function */ if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)())) @@ -305,4 +303,5 @@ H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *f done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5PL__find_plugin_in_cache() */ +#pragma GCC diagnostic pop -- cgit v0.12 From 02a56054b0ddbd6a29e2b09ed39aef0b8e7f001d Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 2 Aug 2017 10:53:17 -0500 Subject: Add h5repack tests for paged aggregation --- tools/test/h5repack/CMakeTests.cmake | 97 ++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 897209c..225f6a8 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -76,6 +76,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_aggr.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_persist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_none.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_persist.h5 # h5diff/testfile ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5 # tools/testfiles @@ -139,6 +145,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/STG.h5repack_none.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/GS.h5repack_paged_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_paged_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SPT.h5repack_aggr.h5.ddl ) foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) @@ -238,7 +250,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_CMP-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else () add_test ( NAME H5REPACK_CMP-${testname} @@ -272,7 +285,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} @@ -305,7 +319,8 @@ # If using memchecker add tests without using scripts add_test ( NAME H5REPACK_DMP-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES DEPENDS ${last_test}) endif () @@ -326,6 +341,40 @@ endif () endmacro () + macro (ADD_H5_STAT_TEST testname testtype resultcode statarg resultfile) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-${testname}-SKIPPED + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile}" + ) + endif () + else () + # If using memchecker add tests without using scripts + add_test ( + NAME H5REPACK_STAT-${testname} + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile} + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPACK_STAT-${testname} PROPERTIES DEPENDS ${last_test}) + endif () + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-h5stat-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=-S;-s;out-${statarg}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${statarg}.${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_STAT-h5stat-${testname} PROPERTIES DEPENDS "H5REPACK_STAT-${testname}") + endif () + endif () + endmacro () + macro (ADD_H5_VERIFY_TEST testname testtype resultcode testfile testdset testfilter) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -972,6 +1021,48 @@ set (TESTTYPE "TEST") ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg}) +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# + set (arg h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_PAGE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_fsm_aggr_persist.h5 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (S_AGGR ${TESTTYPE} 0 S ${arg}) + + set (arg h5repack_none.h5 -S PAGE -T 10 -G 2048) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (STG_PAGE ${TESTTYPE} 0 STG ${arg}) + + set (arg h5repack_paged_nopersist.h5 -G 512 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (GS_AGGR ${TESTTYPE} 0 GS ${arg}) + + set (arg h5repack_paged_persist.h5 -S NONE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_NONE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SPT_FSM_AGGR ${TESTTYPE} 0 SPT ${arg}) + + ######################################################### # layout options (these files have no filters) ######################################################### -- cgit v0.12 From 5615cc2a694516a02731e4d398870385c9736375 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 2 Aug 2017 12:42:40 -0500 Subject: Fix misnamed file in test script --- tools/test/h5dump/testh5dumpxml.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test/h5dump/testh5dumpxml.sh.in b/tools/test/h5dump/testh5dumpxml.sh.in index 39b1361..b3b9e14 100644 --- a/tools/test/h5dump/testh5dumpxml.sh.in +++ b/tools/test/h5dump/testh5dumpxml.sh.in @@ -386,7 +386,7 @@ TOOLTEST torderattr4.h5.xml --xml -H --sort_by=creation_order --sort_order=desce TOOLTEST tfpformat.h5.xml -u -m %.7f tfpformat.h5 # test for HDFFV-10256 issue -TOOLTEST test35.nc --xml test35.nc +TOOLTEST test35.nc.xml --xml test35.nc # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR -- cgit v0.12 From 2332135332f93bf88909d8ef182281c28a8e1818 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 3 Aug 2017 09:17:24 -0500 Subject: changed the test for NAG fortran compiler, (HDFFV-10037) --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index cf82a2e..995d004 100644 --- a/configure.ac +++ b/configure.ac @@ -3516,7 +3516,7 @@ if test "X$HDF_FORTRAN" = "Xyes"; then ### wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) case "`uname`" in Linux*) - if (grep -i 'NAG_Fortran' libtool > /dev/null); then + if test "X$FC_BASENAME" = "Xnagfor"; then cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool fi ;; -- cgit v0.12 From 2a1e499b0134ed319cd9d60a6521ba83b3a2cfa8 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 3 Aug 2017 09:20:05 -0500 Subject: changed comments, (HDFFV-10037) --- configure.ac | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 995d004..91800e7 100644 --- a/configure.ac +++ b/configure.ac @@ -3340,10 +3340,10 @@ fi if test "X$HDF_FORTRAN" = "Xyes"; then - ### libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler + ### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler ### on Linux (other OSs have not been tested). ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to - ### wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) + ### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037) case "`uname`" in Linux*) @@ -3510,10 +3510,10 @@ if test "X$HDF_FORTRAN" = "Xyes"; then ;; esac - ### libtool does not pass the correct argument linking (-WL,-WL,,) for the NAG Fortran compiler + ### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler ### on Linux (other OSs have not been tested). ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to - ### wl="-Wl,-WL,," in the libtool file. (HDFFV-10037) + ### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037) case "`uname`" in Linux*) if test "X$FC_BASENAME" = "Xnagfor"; then -- cgit v0.12