From 44216ac694771788846f216ac75be1b17c44a406 Mon Sep 17 00:00:00 2001 From: songyulu Date: Tue, 15 Feb 2022 11:52:35 -0600 Subject: Fixed some problems with the parallel h5repack. It works with independent open and write of datasets. But the hyperslab write doesn't work yet. --- tools/src/h5repack/Makefile.am | 10 +- tools/src/h5repack/create_h5file.c | 387 +++++++++++++++++++++++++++++++++++++ tools/src/h5repack/h5repack_copy.c | 91 +++++++-- 3 files changed, 469 insertions(+), 19 deletions(-) create mode 100644 tools/src/h5repack/create_h5file.c diff --git a/tools/src/h5repack/Makefile.am b/tools/src/h5repack/Makefile.am index bd3f889..e6acd9a 100644 --- a/tools/src/h5repack/Makefile.am +++ b/tools/src/h5repack/Makefile.am @@ -21,25 +21,21 @@ include $(top_srcdir)/config/commence.am # Include src, test, and tools/lib directories AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -# Always build and test h5repack but build and test h5prepack only if parallel -# is enabled. +# Always build and test h5repack but build and test h5prepack only if parallel # # is enabled. if BUILD_PARALLEL_CONDITIONAL H5PREPACK=h5prepack endif - # A convenience library for the h5repack tool and the h5repack tests noinst_LTLIBRARIES=libh5repack.la libh5repack_la_SOURCES=h5repack.c h5repack_copy.c h5repack_filters.c \ h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ h5repack_verify.c - libh5repack_la_LDFLAGS = $(AM_LDFLAGS) libh5repack_la_LIBADD=$(LIBH5TOOLS) $(LIBHDF5) - -# Our main target, h5repack tool and the conditional h5prepack +# Our main target, h5repack tool and conditional h5prepack bin_PROGRAMS=h5repack $(H5PREPACK) h5repack_SOURCES=h5repack_main.c @@ -47,8 +43,10 @@ h5prepack_SOURCES=h5prepack_main.c # Add h5repack specific linker flags here h5repack_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) +h5prepack_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) # Depend on the hdf5 library, the tools library, the h5repack library h5repack_LDADD=libh5repack.la $(LIBH5TOOLS) $(LIBHDF5) +h5prepack_LDADD=libh5repack.la $(LIBH5TOOLS) $(LIBHDF5) include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5repack/create_h5file.c b/tools/src/h5repack/create_h5file.c new file mode 100644 index 0000000..83c1ac3 --- /dev/null +++ b/tools/src/h5repack/create_h5file.c @@ -0,0 +1,387 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define NAME_LENGTH 1024 +#define ATTR_RANK 1 +#define ONE_RANK 1 +#define DSET_RANK 2 +#define FILENAME "test_h5repack.h5" + +typedef struct { + unsigned int ngroups; + unsigned int nattrs; + unsigned int ndsets; + bool empty_dsets; + unsigned int dset_rows, dset_cols; + unsigned int multiply; + unsigned int attr_dim; + char *filename; +} handler_t; + +static handler_t hand; +int *attr_write; + +/* Show command usage */ +static void +usage(void) +{ + printf(" [-h] [-a] [-c] [-d] [-D] [-e] [-g] [-l] [-m] [-n] [-r]\n\n"); + + printf(" [-h]: this help page\n"); + printf(" [-a]: number of attributes for each group (default is 0)\n"); + printf(" [-c]: number of columns of the smaller datasets (default is 4)\n"); + printf(" [-d]: number of smaller datasets (integer type) under the rot group (default is 0) of which the row & column are set through -r & -c\n"); + printf(" [-e]: empty datasets (no data in the datasets)\n"); + printf(" [-g]: number of groups under the root group (default is 0)\n"); + printf(" [-l]: number of elements (single dimension only) for each attribute (default is 1)\n"); + printf(" [-m]: multiplication factor for dataset dimensions which is applied randomly (default is 1)\n"); + printf(" [-n]: name of the file\n"); + printf(" [-r]: number of rows of the smaller datasets (default is 4)\n"); + + printf("\n"); + printf(" example of usage: ./a.out -c 10 -r 20 -g 4 -d 1 -a 2 -l 4\n"); + printf("\n"); +} + +static bool +state_init(int argc, char **argv) +{ + int opt; + int i; + + hand.ngroups = 0; + hand.nattrs = 0; + hand.ndsets = 0; + hand.empty_dsets = false; + hand.dset_rows = 4; + hand.dset_cols = 4; + hand.multiply = 1; + hand.attr_dim = 1; + hand.filename = strdup(FILENAME); + + while ((opt = getopt(argc, argv, "a:c:d:eg:hl:m:n:r:")) != -1) { + switch (opt) { + case 'a': + if(optarg) { + hand.nattrs = atoi(optarg); + fprintf(stdout, "number of attributes for each group:\t\t%u\n", hand.nattrs); + } else + printf("optarg is null\n"); + + break; + case 'c': + if(optarg) { + hand.dset_cols = atoi(optarg); + fprintf(stdout, "dataset columns:\t\t\t\t%u\n", hand.dset_cols); + } else + printf("optarg is null\n"); + + break; + case 'd': + if(optarg) { + hand.ndsets = atoi(optarg); + fprintf(stdout, "number of datasets:\t\t\t\t%u\n", hand.ndsets); + } else + printf("optarg is null\n"); + + break; + case 'e': + hand.empty_dsets = true; + fprintf(stdout, "empty datasets:\t\t\t\t\ttrue\n"); + + break; + case 'g': + if(optarg) { + hand.ngroups = atoi(optarg); + fprintf(stdout, "number of groups:\t\t\t\t%u\n", hand.ngroups); + } else + printf("optarg is null\n"); + + break; + case 'h': + usage(); + exit(0); + + break; + case 'l': + if(optarg) { + hand.attr_dim = atoi(optarg); + fprintf(stdout, "dimension of attributes:\t\t\t%u\n", hand.attr_dim); + } else + printf("optarg is null\n"); + + break; + case 'm': + if(optarg) { + hand.multiply = atoi(optarg); + fprintf(stdout, "random multiply factor for dataset dimensions:\t%u\n", hand.multiply); + } else + printf("optarg is null\n"); + + break; + case 'n': + if(optarg) { + if(hand.filename) + free(hand.filename); + hand.filename = strdup(optarg); + } else + printf("optarg is null\n"); + + break; + case 'r': + if(optarg) { + hand.dset_rows = atoi(optarg); + fprintf(stdout, "dataset rows:\t\t\t\t\t%u\n", hand.dset_rows); + } else + printf("optarg is null\n"); + + break; + case '?': + default: + usage(); + exit(0); + + break; + } + } + + /* Allocate the memory for the attribute write */ + if (hand.attr_dim) { + attr_write = (int *)malloc((size_t)hand.attr_dim * sizeof(int)); + + /* Initialize the data for the attribute */ + for(i = 0; i < hand.attr_dim; i++) + attr_write[i] = i; + } + + return true; + +error: + return false; +} + +static bool +create_attributes(hid_t group_id) +{ + char attr_name[NAME_LENGTH]; + hsize_t attr_dim[ATTR_RANK]; + hid_t attr_space, attr_id; + int i; + + attr_dim[0] = (hsize_t)hand.attr_dim; + + /* This step could happen earlier for better performance */ + if((attr_space = H5Screate_simple(ATTR_RANK, attr_dim, NULL)) < 0) { + fprintf(stderr, "failed to create memory space for attributes\n"); + goto error; + } + + for (i = 0; i < hand.nattrs; i++) { + sprintf(attr_name, "attr_%d", i + 1); + + if ((attr_id = H5Acreate2(group_id, attr_name, H5T_NATIVE_INT, attr_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + fprintf(stderr, "failed to create attribute '%s'\n", attr_name); + goto error; + } + + if(H5Awrite(attr_id, H5T_NATIVE_INT, attr_write) < 0) { + fprintf(stderr, "failed to write data to the attribute: %s\n", attr_name); + goto error; + } + + if(H5Aclose(attr_id) < 0) { + fprintf(stderr, "failed to close the attribure\n"); + goto error; + } + } + + if(H5Sclose(attr_space) < 0) { + fprintf(stderr, "failed to close the attribure space\n"); + goto error; + } + + return true; + +error: + return false; +} + +static bool +create_groups(hid_t file_id) +{ + int i, j; + hid_t gid; + char gname[NAME_LENGTH]; + hsize_t attr_dim[ATTR_RANK]; + + for (i = 0; i < hand.ngroups; i++) { + sprintf(gname, "group_%d", i + 1); + + if ((gid = H5Gcreate2(file_id, gname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + fprintf(stderr, "couldn't create group '%s'\n", gname); + goto error; + } + + if (hand.nattrs && !create_attributes(gid)) { + fprintf(stderr, "couldn't create attributes for group '%s'\n", gname); + goto error; + } + + if (H5Gclose(gid) < 0) { + fprintf(stderr, "failed to close the file\n"); + goto error; + } + + } + + return true; + +error: + return false; +} + +static bool +create_dsets(hid_t file_id) +{ + int i, j, k; + hid_t dset_id, dset_space, dcpl_id; + char dset_name[NAME_LENGTH]; + hsize_t dimsf[DSET_RANK]; /* dataset dimensions */ + int *dset_data = NULL; + unsigned int multiplication = 1; + + if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + fprintf(stderr, "failed to create the dataset creation property list\n"); + goto error; + } + + /* Allocate space early for testing the parallel h5repack */ + if ((H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY) < 0)) { + fprintf(stderr, "failed to create the dataset creation property list\n"); + goto error; + } + + for (k = 0; k < hand.ndsets; k++) { + sprintf(dset_name, "dset_%d", k + 1); + + multiplication = rand() % 2 ? hand.multiply : 1; + + if (hand.dset_rows == 1) { + dimsf[0] = (hsize_t)hand.dset_cols * multiplication; + + if((dset_space = H5Screate_simple(ONE_RANK, dimsf, NULL)) < 0) { + fprintf(stderr, "failed to create the data space\n"); + goto error; + } + } else { + dimsf[0] = (hsize_t)hand.dset_rows * multiplication; + dimsf[1] = (hsize_t)hand.dset_cols * multiplication; + + if((dset_space = H5Screate_simple(DSET_RANK, dimsf, NULL)) < 0) { + fprintf(stderr, "failed to create the data space\n"); + goto error; + } + } + + dset_data = (int *)malloc(dimsf[0] * dimsf[1] * sizeof(int)); + + if (hand.dset_rows == 1) { + for (i = 0; i < dimsf[0]; i++) + *(dset_data + i) = i; + } else { + for (i = 0; i < dimsf[0]; i++) + for (j = 0; j < dimsf[1]; j++) + *(dset_data + i * dimsf[1] + j) = i + j; + } + + if ((dset_id = H5Dcreate2(file_id, dset_name, H5T_NATIVE_INT, dset_space, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) { + fprintf(stderr, "couldn't create dataset '%s'\n", dset_name); + goto error; + } + + if (!hand.empty_dsets && H5Dwrite(dset_id, H5T_NATIVE_INT, dset_space, dset_space, H5P_DEFAULT, dset_data) < 0) { + fprintf(stderr, "couldn't write data for dataset '%s'\n", dset_name); + goto error; + } + + if (H5Dclose(dset_id) < 0) { + fprintf(stderr, "failed to close the dataset '%s'\n", dset_name); + goto error; + } + + if (H5Sclose(dset_space) < 0) { + fprintf(stderr, "failed to close the dataset space\n"); + goto error; + } + + if (dset_data) + free(dset_data); + } + + if (H5Pclose(dcpl_id) < 0) { + fprintf(stderr, "failed to close the dataset creation property list\n"); + goto error; + } + + return true; + +error: + return false; +} + +static void +release_resources() +{ + if (hand.nattrs && attr_write) + free(attr_write); + + if(hand.filename) + free(hand.filename); +} + +int +main(int argc, char **argv) +{ + hid_t file_id; + + /* seed for random generator */ + srand(time(NULL)); + + if (!state_init(argc, argv)) { + fprintf(stderr, "state_init failed\n"); + goto error; + } + + if ((file_id = H5Fcreate(hand.filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + fprintf(stderr, "failed to create the file '%s'\n", hand.filename); + goto error; + } + + if (hand.ngroups && !create_groups(file_id)) { + fprintf(stderr, "create_groups failed\n"); + goto error; + } + + if (hand.ndsets && !create_dsets(file_id)) { + fprintf(stderr, "create_dsets failed\n"); + goto error; + } + + if (H5Fclose(file_id) < 0) { + fprintf(stderr, "failed to close the file\n"); + goto error; + } + + release_resources(); + + return 0; + +error: + return 1; +} diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index dd49090..8396a61 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -186,8 +186,10 @@ create__dataset(hid_t fidin, hid_t fidout, trav_table_t *travt, size_t index, hb if (!use_h5ocopy) { int j; + /* Dataset is already opened at line 111 - Ray if ((dset_in = H5Dopen2(fidin, travt->objs[index].name, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed"); + */ if (f_space_id == H5I_INVALID_HID) if ((f_space_id = H5Dget_space(dset_in)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dget_space failed"); @@ -308,6 +310,11 @@ create__dataset(hid_t fidin, hid_t fidout, trav_table_t *travt, size_t index, hb * modified dcpl; in that case use the original instead *------------------------------------------------------------------------- */ + + /* For later independent writing of data - Ray */ + if (H5Pset_alloc_time(dcpl_out, H5D_ALLOC_TIME_EARLY) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_alloc_time failed"); + dset_out = H5Dcreate2(fidout, travt->objs[index].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT); @@ -318,6 +325,10 @@ create__dataset(hid_t fidin, hid_t fidout, trav_table_t *travt, size_t index, hb "settings\n", travt->objs[index].name); + /* For later independent writing of data - Ray */ + if (H5Pset_alloc_time(dcpl_in, H5D_ALLOC_TIME_EARLY) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_alloc_time failed"); + if ((dset_out = H5Dcreate2(fidout, travt->objs[index].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dcreate2 failed"); @@ -334,12 +345,20 @@ create__dataset(hid_t fidin, hid_t fidout, trav_table_t *travt, size_t index, hb wtype_id = H5Tcopy(ftype_id); } + /* For later independent writing of data - Ray */ + if (H5Pset_alloc_time(dcpl_in, H5D_ALLOC_TIME_EARLY) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_alloc_time failed"); + if ((dset_out = H5Dcreate2(fidout, travt->objs[index].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dcreate2 failed"); } - H5Dclose(dset_in); + if (dset_in != H5I_INVALID_HID) + H5Dclose(dset_in); + if (dset_out != H5I_INVALID_HID) + H5Dclose(dset_out); /* Ray */ + done: /* Finalize (link) the stack of named datatypes (if any) first * because of reference counting */ @@ -674,7 +693,12 @@ pcreate_new_objects(const char *fnameout, hid_t fcpl, hid_t fidin, hid_t *_fidou if (H5T_REFERENCE == H5Tget_class(ftype_id)) is_ref = 1; - H5Dclose(dset_in); /* No longer needed for this function. */ + /* No longer needed for these functions. */ + if (dset_in != H5I_INVALID_HID && H5Dclose(dset_in) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); + if (H5Tclose(ftype_id) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Tclose failed"); + /*------------------------------------------------------------------------- * check if we should use H5Ocopy or not * if there is a request for filters/layout, we read/write the object @@ -740,9 +764,17 @@ pcreate_new_objects(const char *fnameout, hid_t fcpl, hid_t fidin, hid_t *_fidou } /* for() */ } /* if (travt->objs) */ + /* Reopen the file with MPIO - Ray */ + if (H5Pset_fapl_mpio(options->fout_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + H5TOOLS_GOTO_ERROR((-1), "Unable to select the MPIO driver"); + /* processes (re)open the output file with RDWR access */ - if ((fidout = H5Fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl)) < 0) - H5TOOLS_GOTO_ERROR((-1), "H5Fcreate could not create file <%s>:", fnameout); + //if ((fidout = H5Fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl)) < 0) + // H5TOOLS_GOTO_ERROR((-1), "H5Fcreate could not create file <%s>:", fnameout); + if ((fidout = h5tools_fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl, (options->fout_fapl != H5P_DEFAULT), + NULL, (size_t)0)) < 0) + H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnameout, H5FOPENERROR); + /* Return the outputput file descriptor */ *_fidout = fidout; @@ -825,7 +857,7 @@ check_dataset_sizes(hid_t fidin, trav_table_t *travt) travt->objs[i].use_hyperslab = true; if (H5Sclose(f_space_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Sclose failed"); - if (H5Dclose(dset_in) < 0) + if (dset_in != H5I_INVALID_HID && H5Dclose(dset_in) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); } } @@ -864,6 +896,7 @@ select_objs_by_rank(hid_t fidin, trav_table_t *orig, int **table_index) index = (int *)HDcalloc(orig->nobjs, sizeof(int)); for (k = 0; k < (int)orig->nobjs; k++) { int mod_rank = (int)((int)k % g_nTasks); + if ((orig->objs[k].use_hyperslab) || (mod_rank == (int)g_nID)) { index[count++] = k; } @@ -906,6 +939,18 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) int ret_value = 0; int g_ret; + /* Create a FAPL and make open call to the input file with MPIO - Ray */ + if ((options->fin_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + H5TOOLS_GOTO_ERROR((-1), "Unable to create fapl for the input file"); + + /* open the file with MPIO - Ray */ +#ifdef H5_HAVE_PARALLEL +/* if (g_Parallel) { + if (H5Pset_fapl_mpio(options->fin_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + H5TOOLS_GOTO_ERROR((-1), "Unable to select the MPIO driver"); + }*/ +#endif + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); @@ -1179,9 +1224,23 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) if (pcreate_new_objects(fnameout, fcpl, fidin, &fidout, travt, options) < 0) H5TOOLS_GOTO_ERROR((-1), "pcreate_new_objects from <%s> into <%s>", fnamein, fnameout); + if (H5Fclose(fidin) < 0) + H5TOOLS_GOTO_ERROR((-1), "failed to close the input file"); + + /* reopen the file with MPIO - Ray */ + if (H5Pset_fapl_mpio(options->fin_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + H5TOOLS_GOTO_ERROR((-1), "Unable to select the MPIO driver"); + + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + NULL, (size_t)0)) < 0) + H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); + if (pcopy_objects(fidin, fidout, travt, travt_indices, tiCount, options) < 0) H5TOOLS_GOTO_ERROR((-1), "pcopy_objects from <%s> could not copy data to <%s>", fnamein, fnameout); + + ptools_barrier(); + if (pcopy_refobjs(fidin, fidout, travt, travt_indices, tiCount, options) < 0) H5TOOLS_GOTO_ERROR((-1), "pcopy_refobjs from <%s> could not copy data to <%s>", fnamein, fnameout); @@ -1215,8 +1274,11 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) if (H5Pclose(fcpl) < 0) H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); if (H5Pclose(options->fout_fapl) < 0) - H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + H5TOOLS_GOTO_ERROR((-1), "could not close fapl"); options->fout_fapl = H5P_DEFAULT; + if (H5Pclose(options->fin_fapl) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fapl"); + options->fin_fapl = H5P_DEFAULT; if (H5Pclose(gcpl_in) < 0) H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); if (H5Gclose(grp_in) < 0) @@ -1314,7 +1376,8 @@ done: * the boundary would be dataset's dims. * * The calculation starts from the last dimension (h5dump dims output). - *-----------------------------------------*/ + *----------------------------------------- + */ int get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t size_datum, @@ -1455,8 +1518,8 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */ hid_t dxpl_id = H5P_DEFAULT; /* dataset transfer property list */ named_dt_t * named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ - size_t msize; /* size of type */ - hsize_t nelmts; /* number of elements in dataset */ + /* size_t msize; size of type */ + /* hsize_t nelmts; number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ hsize_t dsize_in; /* input dataset size before filter */ @@ -1510,6 +1573,7 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in * Copy attrs manually *------------------------------------------------------------------------- */ + /* Open the dataset independently - Ray */ if ((dset_in = H5Dopen2(fidin, travt->objs[obj_i].name, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed"); if ((dset_out = H5Dopen2(fidout, travt->objs[obj_i].name, H5P_DEFAULT)) < 0) @@ -1530,8 +1594,8 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in dsize_in = H5Dget_storage_size(dset_in); if (travt->objs[obj_i].use_hyperslab) { - size_t p_type_nbytes = msize; /*size of memory type */ - hsize_t p_nelmts = nelmts; /*total elements */ + size_t p_type_nbytes; /*size of memory type */ + hsize_t p_nelmts; /*total elements */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ unsigned int vl_data = 0; /*contains VL datatypes */ @@ -1602,6 +1666,7 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in HDmemset(zero, 0, sizeof zero); hs_sel_offset[0] = hs_context->hs_offset[0]; + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_select_nelmts) { if (rank > 0) { /* calculate the hyperslab selections. @@ -1691,10 +1756,10 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in HDfree(dataBuf); } - if (H5Dclose(dset_in) < 0) + if (dset_in != H5I_INVALID_HID && H5Dclose(dset_in) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); dset_in = H5I_INVALID_HID; - if (H5Dclose(dset_out) < 0) + if (dset_out != H5I_INVALID_HID && H5Dclose(dset_out) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); dset_out = H5I_INVALID_HID; -- cgit v0.12 From 7bce768a31af48cd6a940ed7ed21cd39377762a8 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 17:57:37 +0000 Subject: Committing clang-format changes --- tools/src/h5repack/create_h5file.c | 212 ++++++++++++++++++++----------------- tools/src/h5repack/h5repack_copy.c | 57 +++++----- 2 files changed, 141 insertions(+), 128 deletions(-) diff --git a/tools/src/h5repack/create_h5file.c b/tools/src/h5repack/create_h5file.c index 83c1ac3..5efa325 100644 --- a/tools/src/h5repack/create_h5file.c +++ b/tools/src/h5repack/create_h5file.c @@ -7,11 +7,11 @@ #include #include -#define NAME_LENGTH 1024 -#define ATTR_RANK 1 -#define ONE_RANK 1 -#define DSET_RANK 2 -#define FILENAME "test_h5repack.h5" +#define NAME_LENGTH 1024 +#define ATTR_RANK 1 +#define ONE_RANK 1 +#define DSET_RANK 2 +#define FILENAME "test_h5repack.h5" typedef struct { unsigned int ngroups; @@ -21,11 +21,11 @@ typedef struct { unsigned int dset_rows, dset_cols; unsigned int multiply; unsigned int attr_dim; - char *filename; + char * filename; } handler_t; static handler_t hand; -int *attr_write; +int * attr_write; /* Show command usage */ static void @@ -36,11 +36,13 @@ usage(void) printf(" [-h]: this help page\n"); printf(" [-a]: number of attributes for each group (default is 0)\n"); printf(" [-c]: number of columns of the smaller datasets (default is 4)\n"); - printf(" [-d]: number of smaller datasets (integer type) under the rot group (default is 0) of which the row & column are set through -r & -c\n"); + printf(" [-d]: number of smaller datasets (integer type) under the rot group (default is 0) of which " + "the row & column are set through -r & -c\n"); printf(" [-e]: empty datasets (no data in the datasets)\n"); printf(" [-g]: number of groups under the root group (default is 0)\n"); printf(" [-l]: number of elements (single dimension only) for each attribute (default is 1)\n"); - printf(" [-m]: multiplication factor for dataset dimensions which is applied randomly (default is 1)\n"); + printf( + " [-m]: multiplication factor for dataset dimensions which is applied randomly (default is 1)\n"); printf(" [-n]: name of the file\n"); printf(" [-r]: number of rows of the smaller datasets (default is 4)\n"); @@ -55,95 +57,103 @@ state_init(int argc, char **argv) int opt; int i; - hand.ngroups = 0; - hand.nattrs = 0; - hand.ndsets = 0; + hand.ngroups = 0; + hand.nattrs = 0; + hand.ndsets = 0; hand.empty_dsets = false; - hand.dset_rows = 4; - hand.dset_cols = 4; - hand.multiply = 1; - hand.attr_dim = 1; - hand.filename = strdup(FILENAME); + hand.dset_rows = 4; + hand.dset_cols = 4; + hand.multiply = 1; + hand.attr_dim = 1; + hand.filename = strdup(FILENAME); while ((opt = getopt(argc, argv, "a:c:d:eg:hl:m:n:r:")) != -1) { switch (opt) { - case 'a': - if(optarg) { + case 'a': + if (optarg) { hand.nattrs = atoi(optarg); fprintf(stdout, "number of attributes for each group:\t\t%u\n", hand.nattrs); - } else + } + else printf("optarg is null\n"); - break; - case 'c': - if(optarg) { + break; + case 'c': + if (optarg) { hand.dset_cols = atoi(optarg); fprintf(stdout, "dataset columns:\t\t\t\t%u\n", hand.dset_cols); - } else + } + else printf("optarg is null\n"); - break; - case 'd': - if(optarg) { + break; + case 'd': + if (optarg) { hand.ndsets = atoi(optarg); fprintf(stdout, "number of datasets:\t\t\t\t%u\n", hand.ndsets); - } else + } + else printf("optarg is null\n"); - break; - case 'e': - hand.empty_dsets = true; - fprintf(stdout, "empty datasets:\t\t\t\t\ttrue\n"); + break; + case 'e': + hand.empty_dsets = true; + fprintf(stdout, "empty datasets:\t\t\t\t\ttrue\n"); - break; - case 'g': - if(optarg) { + break; + case 'g': + if (optarg) { hand.ngroups = atoi(optarg); fprintf(stdout, "number of groups:\t\t\t\t%u\n", hand.ngroups); - } else + } + else printf("optarg is null\n"); - break; - case 'h': - usage(); - exit(0); + break; + case 'h': + usage(); + exit(0); - break; - case 'l': - if(optarg) { + break; + case 'l': + if (optarg) { hand.attr_dim = atoi(optarg); fprintf(stdout, "dimension of attributes:\t\t\t%u\n", hand.attr_dim); - } else + } + else printf("optarg is null\n"); - break; - case 'm': - if(optarg) { + break; + case 'm': + if (optarg) { hand.multiply = atoi(optarg); fprintf(stdout, "random multiply factor for dataset dimensions:\t%u\n", hand.multiply); - } else + } + else printf("optarg is null\n"); - break; - case 'n': - if(optarg) { - if(hand.filename) + break; + case 'n': + if (optarg) { + if (hand.filename) free(hand.filename); hand.filename = strdup(optarg); - } else + } + else printf("optarg is null\n"); - break; - case 'r': - if(optarg) { + break; + case 'r': + if (optarg) { hand.dset_rows = atoi(optarg); fprintf(stdout, "dataset rows:\t\t\t\t\t%u\n", hand.dset_rows); - } else + } + else printf("optarg is null\n"); - break; - case '?': - default: + break; + case '?': + default: usage(); exit(0); @@ -156,7 +166,7 @@ state_init(int argc, char **argv) attr_write = (int *)malloc((size_t)hand.attr_dim * sizeof(int)); /* Initialize the data for the attribute */ - for(i = 0; i < hand.attr_dim; i++) + for (i = 0; i < hand.attr_dim; i++) attr_write[i] = i; } @@ -169,42 +179,43 @@ error: static bool create_attributes(hid_t group_id) { - char attr_name[NAME_LENGTH]; - hsize_t attr_dim[ATTR_RANK]; - hid_t attr_space, attr_id; - int i; + char attr_name[NAME_LENGTH]; + hsize_t attr_dim[ATTR_RANK]; + hid_t attr_space, attr_id; + int i; attr_dim[0] = (hsize_t)hand.attr_dim; /* This step could happen earlier for better performance */ - if((attr_space = H5Screate_simple(ATTR_RANK, attr_dim, NULL)) < 0) { + if ((attr_space = H5Screate_simple(ATTR_RANK, attr_dim, NULL)) < 0) { fprintf(stderr, "failed to create memory space for attributes\n"); goto error; } - + for (i = 0; i < hand.nattrs; i++) { sprintf(attr_name, "attr_%d", i + 1); - if ((attr_id = H5Acreate2(group_id, attr_name, H5T_NATIVE_INT, attr_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + if ((attr_id = + H5Acreate2(group_id, attr_name, H5T_NATIVE_INT, attr_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) { fprintf(stderr, "failed to create attribute '%s'\n", attr_name); goto error; } - if(H5Awrite(attr_id, H5T_NATIVE_INT, attr_write) < 0) { + if (H5Awrite(attr_id, H5T_NATIVE_INT, attr_write) < 0) { fprintf(stderr, "failed to write data to the attribute: %s\n", attr_name); goto error; - } + } - if(H5Aclose(attr_id) < 0) { + if (H5Aclose(attr_id) < 0) { fprintf(stderr, "failed to close the attribure\n"); goto error; - } + } } - if(H5Sclose(attr_space) < 0) { + if (H5Sclose(attr_space) < 0) { fprintf(stderr, "failed to close the attribure space\n"); goto error; - } + } return true; @@ -215,10 +226,10 @@ error: static bool create_groups(hid_t file_id) { - int i, j; - hid_t gid; - char gname[NAME_LENGTH]; - hsize_t attr_dim[ATTR_RANK]; + int i, j; + hid_t gid; + char gname[NAME_LENGTH]; + hsize_t attr_dim[ATTR_RANK]; for (i = 0; i < hand.ngroups; i++) { sprintf(gname, "group_%d", i + 1); @@ -236,8 +247,7 @@ create_groups(hid_t file_id) if (H5Gclose(gid) < 0) { fprintf(stderr, "failed to close the file\n"); goto error; - } - + } } return true; @@ -249,23 +259,23 @@ error: static bool create_dsets(hid_t file_id) { - int i, j, k; - hid_t dset_id, dset_space, dcpl_id; - char dset_name[NAME_LENGTH]; - hsize_t dimsf[DSET_RANK]; /* dataset dimensions */ - int *dset_data = NULL; + int i, j, k; + hid_t dset_id, dset_space, dcpl_id; + char dset_name[NAME_LENGTH]; + hsize_t dimsf[DSET_RANK]; /* dataset dimensions */ + int * dset_data = NULL; unsigned int multiplication = 1; if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) { fprintf(stderr, "failed to create the dataset creation property list\n"); goto error; - } + } - /* Allocate space early for testing the parallel h5repack */ + /* Allocate space early for testing the parallel h5repack */ if ((H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY) < 0)) { fprintf(stderr, "failed to create the dataset creation property list\n"); goto error; - } + } for (k = 0; k < hand.ndsets; k++) { sprintf(dset_name, "dset_%d", k + 1); @@ -275,15 +285,16 @@ create_dsets(hid_t file_id) if (hand.dset_rows == 1) { dimsf[0] = (hsize_t)hand.dset_cols * multiplication; - if((dset_space = H5Screate_simple(ONE_RANK, dimsf, NULL)) < 0) { + if ((dset_space = H5Screate_simple(ONE_RANK, dimsf, NULL)) < 0) { fprintf(stderr, "failed to create the data space\n"); goto error; } - } else { + } + else { dimsf[0] = (hsize_t)hand.dset_rows * multiplication; dimsf[1] = (hsize_t)hand.dset_cols * multiplication; - if((dset_space = H5Screate_simple(DSET_RANK, dimsf, NULL)) < 0) { + if ((dset_space = H5Screate_simple(DSET_RANK, dimsf, NULL)) < 0) { fprintf(stderr, "failed to create the data space\n"); goto error; } @@ -294,18 +305,21 @@ create_dsets(hid_t file_id) if (hand.dset_rows == 1) { for (i = 0; i < dimsf[0]; i++) *(dset_data + i) = i; - } else { + } + else { for (i = 0; i < dimsf[0]; i++) for (j = 0; j < dimsf[1]; j++) *(dset_data + i * dimsf[1] + j) = i + j; } - if ((dset_id = H5Dcreate2(file_id, dset_name, H5T_NATIVE_INT, dset_space, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) { + if ((dset_id = H5Dcreate2(file_id, dset_name, H5T_NATIVE_INT, dset_space, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT)) < 0) { fprintf(stderr, "couldn't create dataset '%s'\n", dset_name); goto error; } - if (!hand.empty_dsets && H5Dwrite(dset_id, H5T_NATIVE_INT, dset_space, dset_space, H5P_DEFAULT, dset_data) < 0) { + if (!hand.empty_dsets && + H5Dwrite(dset_id, H5T_NATIVE_INT, dset_space, dset_space, H5P_DEFAULT, dset_data) < 0) { fprintf(stderr, "couldn't write data for dataset '%s'\n", dset_name); goto error; } @@ -313,12 +327,12 @@ create_dsets(hid_t file_id) if (H5Dclose(dset_id) < 0) { fprintf(stderr, "failed to close the dataset '%s'\n", dset_name); goto error; - } + } if (H5Sclose(dset_space) < 0) { fprintf(stderr, "failed to close the dataset space\n"); goto error; - } + } if (dset_data) free(dset_data); @@ -327,7 +341,7 @@ create_dsets(hid_t file_id) if (H5Pclose(dcpl_id) < 0) { fprintf(stderr, "failed to close the dataset creation property list\n"); goto error; - } + } return true; @@ -341,7 +355,7 @@ release_resources() if (hand.nattrs && attr_write) free(attr_write); - if(hand.filename) + if (hand.filename) free(hand.filename); } @@ -376,7 +390,7 @@ main(int argc, char **argv) if (H5Fclose(file_id) < 0) { fprintf(stderr, "failed to close the file\n"); goto error; - } + } release_resources(); diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 8396a61..1da2ade 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -186,7 +186,7 @@ create__dataset(hid_t fidin, hid_t fidout, trav_table_t *travt, size_t index, hb if (!use_h5ocopy) { int j; - /* Dataset is already opened at line 111 - Ray + /* Dataset is already opened at line 111 - Ray if ((dset_in = H5Dopen2(fidin, travt->objs[index].name, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed"); */ @@ -769,13 +769,12 @@ pcreate_new_objects(const char *fnameout, hid_t fcpl, hid_t fidin, hid_t *_fidou H5TOOLS_GOTO_ERROR((-1), "Unable to select the MPIO driver"); /* processes (re)open the output file with RDWR access */ - //if ((fidout = H5Fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl)) < 0) + // if ((fidout = H5Fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl)) < 0) // H5TOOLS_GOTO_ERROR((-1), "H5Fcreate could not create file <%s>:", fnameout); - if ((fidout = h5tools_fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl, (options->fout_fapl != H5P_DEFAULT), - NULL, (size_t)0)) < 0) + if ((fidout = h5tools_fopen(fnameout, H5F_ACC_RDWR, options->fout_fapl, + (options->fout_fapl != H5P_DEFAULT), NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnameout, H5FOPENERROR); - /* Return the outputput file descriptor */ *_fidout = fidout; @@ -943,7 +942,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) if ((options->fin_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) H5TOOLS_GOTO_ERROR((-1), "Unable to create fapl for the input file"); - /* open the file with MPIO - Ray */ + /* open the file with MPIO - Ray */ #ifdef H5_HAVE_PARALLEL /* if (g_Parallel) { if (H5Pset_fapl_mpio(options->fin_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) @@ -1231,8 +1230,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) if (H5Pset_fapl_mpio(options->fin_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) H5TOOLS_GOTO_ERROR((-1), "Unable to select the MPIO driver"); - if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), - NULL, (size_t)0)) < 0) + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, + (options->fin_fapl != H5P_DEFAULT), NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); if (pcopy_objects(fidin, fidout, travt, travt_indices, tiCount, options) < 0) @@ -1509,24 +1508,24 @@ int pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, int obj_count, pack_opt_t *options) { - hid_t dset_in = H5I_INVALID_HID; /* read dataset ID */ - hid_t dset_out = H5I_INVALID_HID; /* write dataset ID */ - hid_t dcpl_in = H5I_INVALID_HID; /* dataset creation property list ID */ - hid_t dcpl_out = H5I_INVALID_HID; /* dataset creation property list ID */ - hid_t f_space_id = H5I_INVALID_HID; /* file space ID */ - hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ - hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */ - hid_t dxpl_id = H5P_DEFAULT; /* dataset transfer property list */ - named_dt_t * named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ + hid_t dset_in = H5I_INVALID_HID; /* read dataset ID */ + hid_t dset_out = H5I_INVALID_HID; /* write dataset ID */ + hid_t dcpl_in = H5I_INVALID_HID; /* dataset creation property list ID */ + hid_t dcpl_out = H5I_INVALID_HID; /* dataset creation property list ID */ + hid_t f_space_id = H5I_INVALID_HID; /* file space ID */ + hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ + hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */ + hid_t dxpl_id = H5P_DEFAULT; /* dataset transfer property list */ + named_dt_t *named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ /* size_t msize; size of type */ /* hsize_t nelmts; number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ - hsize_t dsize_in; /* input dataset size before filter */ - void * buf = NULL; /* buffer for raw data */ - void * hslab_buf = NULL; /* hyperslab buffer for raw data */ - H5_timer_t timer; /* Timer for read/write operations */ - H5_timevals_t times; /* Elapsed time for each operation */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ + void * buf = NULL; /* buffer for raw data */ + void * hslab_buf = NULL; /* hyperslab buffer for raw data */ + H5_timer_t timer; /* Timer for read/write operations */ + H5_timevals_t times; /* Elapsed time for each operation */ static double read_time = 0; static double write_time = 0; h5tool_link_info_t linkinfo; @@ -1594,11 +1593,11 @@ pcopy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int *obj_index, in dsize_in = H5Dget_storage_size(dset_in); if (travt->objs[obj_i].use_hyperslab) { - size_t p_type_nbytes; /*size of memory type */ - hsize_t p_nelmts; /*total elements */ - hsize_t elmtno; /*counter */ - int carry; /*counter carry value */ - unsigned int vl_data = 0; /*contains VL datatypes */ + size_t p_type_nbytes; /*size of memory type */ + hsize_t p_nelmts; /*total elements */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ /* hyperslab info */ hsize_t hslab_dims[H5S_MAX_RANK]; /*hyperslab dims */ -- cgit v0.12