diff options
Diffstat (limited to 'tools/src/h5repack')
-rw-r--r-- | tools/src/h5repack/h5repack.c | 92 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack.h | 8 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack_copy.c | 149 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack_filters.c | 4 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack_main.c | 118 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack_refs.c | 11 | ||||
-rw-r--r-- | tools/src/h5repack/h5repack_verify.c | 10 |
7 files changed, 286 insertions, 106 deletions
diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index 7cad36b..d0aad79 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -221,6 +221,7 @@ h5repack_addlayout(const char *str, pack_opt_t *options) * to free the stack. *------------------------------------------------------------------------- */ + hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) @@ -354,24 +355,28 @@ done: int copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) { - hid_t attr_id = H5I_INVALID_HID; /* attr ID */ - hid_t attr_out = H5I_INVALID_HID; /* attr ID */ - hid_t space_id = H5I_INVALID_HID; /* space ID */ - hid_t ftype_id = H5I_INVALID_HID; /* file type ID */ - hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ - size_t msize; /* size of type */ - void * buf = NULL; /* data buffer */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - htri_t is_named; /* Whether the datatype is named */ - hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ - char name[255]; - H5O_info2_t oinfo; /* object info */ - int j; - unsigned u; - hbool_t is_ref = 0; - H5T_class_t type_class = -1; - int ret_value = 0; + hid_t attr_id = H5I_INVALID_HID; /* attr ID */ + hid_t attr_out = H5I_INVALID_HID; /* attr ID */ + hid_t space_id = H5I_INVALID_HID; /* space ID */ + hid_t ftype_id = H5I_INVALID_HID; /* file type ID */ + hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ + size_t msize; /* size of type */ + void * buf = NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + htri_t is_named; /* Whether the datatype is named */ + hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ + H5_timer_t timer; /* Timer for read/write operations */ + H5_timevals_t times; /* Elapsed time for each operation */ + static double read_time = 0; + static double write_time = 0; + char name[255]; + H5O_info2_t oinfo; /* object info */ + int j; + unsigned u; + hbool_t is_ref = 0; + H5T_class_t type_class = -1; + int ret_value = 0; if (H5Oget_info3(loc_in, &oinfo, H5O_INFO_NUM_ATTRS) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Oget_info failed"); @@ -468,6 +473,9 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ } /* end for each member */ } /* end if type_class is H5T_COMPOUND */ + read_time = 0; + write_time = 0; + if (!is_ref) { /*----------------------------------------------------------------- * read to memory @@ -478,8 +486,17 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ if (buf == NULL) { H5TOOLS_GOTO_ERROR((-1), "HDmalloc failed"); } /* end if */ + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Aread(attr_id, wtype_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Aread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + } /*----------------------------------------------------------------- * copy @@ -488,8 +505,18 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ if ((attr_out = H5Acreate2(loc_out, name, wtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Acreate2 failed on ,%s>", name); + + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Awrite(attr_out, wtype_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Awrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /*close*/ if (H5Aclose(attr_out) < 0) @@ -499,11 +526,14 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ * be reclaimed */ if (TRUE == h5tools_detect_vlen(wtype_id)) H5Treclaim(wtype_id, space_id, H5P_DEFAULT, buf); + HDfree(buf); buf = NULL; } /*H5T_REFERENCE*/ - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_ATTR_TIME, "attr", read_time, write_time, name); + else HDprintf(FORMAT_OBJ_ATTR, "attr", name); /*--------------------------------------------------------------------- @@ -569,7 +599,7 @@ check_options(pack_opt_t *options) * Objects to layout *------------------------------------------------------------------------- */ - if (options->verbose && have_request(options)) { + if (options->verbose > 0 && have_request(options)) { if (options->all_layout == 1) { HDprintf("All objects to modify layout are...\n"); switch (options->layout_g) { @@ -610,7 +640,7 @@ check_options(pack_opt_t *options) char *name = options->op_tbl->objs[i].path; if (options->op_tbl->objs[i].chunk.rank > 0) { - if (options->verbose) { + if (options->verbose > 0) { HDprintf(" <%s> with chunk size ", name); for (k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) HDprintf("%d ", (int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); @@ -619,7 +649,7 @@ check_options(pack_opt_t *options) has_ck = 1; } else if (options->op_tbl->objs[i].chunk.rank == -2) { /* TODO: replace 'magic number' */ - if (options->verbose) + if (options->verbose > 0) HDprintf(" <%s> %s\n", name, "NONE (contiguous)"); has_ck = 1; } @@ -633,7 +663,7 @@ check_options(pack_opt_t *options) *------------------------------------------------------------------------- */ - if (options->verbose && have_request(options)) { + if (options->verbose > 0 && have_request(options)) { if (options->all_filter == 1) { HDprintf("All objects to apply filter are...\n"); for (k = 0; k < options->n_filter_g; k++) { @@ -670,7 +700,7 @@ check_options(pack_opt_t *options) char * name = pack.path; for (j = 0; j < pack.nfilters; j++) { - if (options->verbose) { + if (options->verbose > 0) { if (pack.filter[j].filtn >= 0) { if (pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET) { HDprintf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), @@ -708,7 +738,7 @@ check_options(pack_opt_t *options) *------------------------------------------------------------------------ */ if (options->ublock_filename != NULL && options->ublock_size == 0) { - if (options->verbose) { + if (options->verbose > 0) { HDprintf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename); options->ublock_size = 1024; @@ -758,8 +788,8 @@ check_objects(const char *fname, pack_opt_t *options) * open the file *------------------------------------------------------------------------- */ - if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, - (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + NULL, 0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname, H5FOPENERROR); /*------------------------------------------------------------------------- @@ -781,21 +811,21 @@ check_objects(const char *fname, pack_opt_t *options) *------------------------------------------------------------------------- */ - if (options->verbose) + if (options->verbose > 0) HDprintf("Opening file. Searching %zu objects to modify ...\n", travt->nobjs); for (i = 0; i < options->op_tbl->nelems; i++) { pack_info_t obj = options->op_tbl->objs[i]; char * name = obj.path; - if (options->verbose) + if (options->verbose > 0) HDprintf(" <%s>", name); /* the input object names are present in the file and are valid */ if (h5trav_getindext(name, travt) < 0) H5TOOLS_GOTO_ERROR((-1), "%s Could not find <%s> in file <%s>. Exiting...\n", - (options->verbose ? "\n" : ""), name, fname); - if (options->verbose) + (options->verbose > 0 ? "\n" : ""), name, fname); + if (options->verbose > 0) HDprintf("...Found\n"); for (ifil = 0; ifil < obj.nfilters; ifil++) { diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index 74525f4..65e5d40 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -27,6 +27,12 @@ #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ #define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ +/* timing formats */ +#define FORMAT_OBJ_TIME " %-27s %e/%e %s\n" /* obj type, name */ +#define FORMAT_OBJ_ATTR_TIME " %-27s %e/%e %s\n" /* obj type, name */ +#define FORMAT_OBJ_NOTIME " %-27s %s\n" /* obj type, name */ +#define FORMAT_OBJ_ATTR_NOTIME " %-27s %s\n" /* obj type, name */ + /* File space default information */ #define FS_PAGESIZE_DEF 4096 #define FS_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR @@ -118,7 +124,7 @@ typedef struct { int grp_compact; /* Set the maximum number of links to store as header messages in the group */ int grp_indexed; /* Set the minimum number of links to store in the indexed format */ int msg_size[8]; /* Minimum size of shared messages: dataspace, - datatype, fill value, filter pipleline, attribute */ + datatype, fill value, filter pipeline, attribute */ const char * ublock_filename; /* user block file name */ hsize_t ublock_size; /* user block size */ hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 934b4d1..c38a431 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -39,7 +39,8 @@ */ static int get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t size_datum, hsize_t dims_hslab[], hsize_t *hslab_nbytes_p); -static void print_dataset_info(hid_t dcpl_id, char *objname, double per, int pr); +static void print_dataset_info(hid_t dcpl_id, char *objname, double per, int pr, pack_opt_t *options, + double read_time, double write_time); static int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *options); static int copy_user_block(const char *infile, const char *outfile, hsize_t size); #if defined(H5REPACK_DEBUG_USER_BLOCK) @@ -66,7 +67,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) hid_t fcpl = H5P_DEFAULT; /* file creation property list ID */ trav_table_t * travt = NULL; hsize_t ub_size = 0; /* size of user block */ - H5F_fspace_strategy_t set_strategy; /* Strategy to be set in outupt file */ + H5F_fspace_strategy_t set_strategy; /* Strategy to be set in output file */ hbool_t set_persist; /* Persist free-space status to be set in output file */ hsize_t set_threshold; /* Free-space section threshold to be set in output file */ hsize_t set_pagesize; /* File space page size to be set in output file */ @@ -81,8 +82,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) * open input file *------------------------------------------------------------------------- */ - if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, - (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, (size_t)0)) < 0) + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); /* get user block size and file space strategy/persist/threshold */ @@ -187,7 +188,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) H5TOOLS_GOTO_ERROR((-1), "H5Pset_shared_mesg_nindexes failed to set the number of shared " "object header message indexes"); - /* msg_size[0]=dataspace, 1=datatype, 2=file value, 3=filter pipleline, 4=attribute */ + /* msg_size[0]=dataspace, 1=datatype, 2=file value, 3=filter pipeline, 4=attribute */ for (i = 0; i < (nindex - 1); i++) if (H5Pset_shared_mesg_index(fcpl, i, mesg_type_flags[i], min_mesg_sizes[i]) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pset_shared_mesg_index failed to configure the specified " @@ -298,7 +299,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) * create the output file *------------------------------------------------------------------------- */ - if (options->verbose) + if (options->verbose > 0) HDprintf("Making new file ...\n"); if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, options->fout_fapl)) < 0) @@ -401,18 +402,18 @@ done: /*------------------------------------------------------------------------- * Function: get_hyperslab * - * Purpose: Calulate a hyperslab from a dataset for higher performance. - * The size of hyperslab is limitted by H5TOOLS_BUFSIZE. - * Return the hyperslab dimentions and size in byte. + * Purpose: Calculate a hyperslab from a dataset for higher performance. + * The size of hyperslab is limited by H5TOOLS_BUFSIZE. + * Return the hyperslab dimensions and size in byte. * * Return: 0 - SUCCEED, -1 FAILED * * Parameters: * dcpl_id : [IN] dataset creation property. * rank_dset : [IN] dataset rank - * dims_dset[] : [IN] dataset dimentions + * dims_dset[] : [IN] dataset dimensions * size_datum : [IN] size of a data element in byte - * dims_hslab[] : [OUT] calculated hyperslab dimentions + * dims_hslab[] : [OUT] calculated hyperslab dimensions * * hslab_nbytes_p : [OUT] total byte of the hyperslab * * Update: @@ -427,7 +428,7 @@ done: * 3. If not chunked, each data element would be a unit of collection and * the boundary would be dataset's dims. * - * The calulation starts from the last dimention (h5dump dims output). + * The calculation starts from the last dimension (h5dump dims output). *-----------------------------------------*/ int @@ -441,8 +442,8 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si hsize_t size_chunk = 1; hsize_t nchunk_fit; /* number of chunks that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ hsize_t ndatum_fit; /* number of dataum that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ - hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimentions */ - hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimentions */ + hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimensions */ + hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimensions */ hsize_t hslab_nbytes; /* size of hyperslab in byte */ int ret_value = 0; @@ -467,18 +468,18 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* 1. if a chunk fit in hyperslab buffer */ if (nchunk_fit >= 1) { - /* Calulate a hyperslab that contains as many chunks that can fit + /* Calculate a hyperslab that contains as many chunks that can fit * in hyperslab buffer. Hyperslab will be increased starting from - * the last dimention of the dataset (see h5dump's dims output). + * the last dimension of the dataset (see h5dump's dims output). * The calculation boundary is dataset dims. * In the loop, used mapping from a datum to a chunk to figure out * chunk based hyperslab. */ for (k = rank_dset; k > 0; --k) { - /* map dataset dimentions with a chunk dims */ + /* map dataset dimensions with a chunk dims */ chunk_dims_map[k - 1] = dims_dset[k - 1] / dims_chunk[k - 1]; - /* if reminder exist, increse by 1 to cover partial edge chunks */ + /* if reminder exist, increase by 1 to cover partial edge chunks */ if (dims_dset[k - 1] % dims_chunk[k - 1] > 0) chunk_dims_map[k - 1]++; @@ -491,7 +492,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si if (nchunk_fit == 0) nchunk_fit = 1; - /* get hyperslab dimentions as unmapping to actual size */ + /* get hyperslab dimensions as unmapping to actual size */ dims_hslab[k - 1] = MIN((hs_dims_map[k - 1] * dims_chunk[k - 1]), dims_dset[k - 1]); /* calculate total size for the hyperslab */ @@ -500,9 +501,9 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si } /* 2. if a chunk is bigger than hyperslab buffer */ else { - /* Calulate a hyperslab that contains as many data elements that + /* Calculate a hyperslab that contains as many data elements that * can fit in hyperslab buffer. Hyperslab will be increased - * starting from the last dimention of the chunk (see h5dump's dims + * starting from the last dimension of the chunk (see h5dump's dims * output). * The calculation boundary is a chunk dims. */ @@ -512,7 +513,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* if a datum is bigger than rest of buffer */ if (ndatum_fit == 0) ndatum_fit = 1; - /* get hyperslab dimentions within a chunk boundary */ + /* get hyperslab dimensions within a chunk boundary */ dims_hslab[k - 1] = MIN(dims_chunk[k - 1], ndatum_fit); /* calculate total size for the hyperslab */ @@ -525,9 +526,9 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si } /* 3. if dataset is not chunked */ else { - /* Calulate a hyperslab that contains as many data elements that can + /* Calculate a hyperslab that contains as many data elements that can * fit in hyperslab buffer. Hyperslab will be increased starting from - * the last dimention of the dataset (see h5dump's dims output). + * the last dimension of the dataset (see h5dump's dims output). * The calculation boundary is dataset dims. */ for (k = rank_dset; k > 0; --k) { @@ -536,7 +537,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* if a datum is bigger than rest of buffer */ if (ndatum_fit == 0) ndatum_fit = 1; - /* get hyperslab dimentions within dataset boundary */ + /* get hyperslab dimensions within dataset boundary */ dims_hslab[k - 1] = MIN(dims_dset[k - 1], ndatum_fit); /* calculate total size for the hyperslab */ @@ -600,7 +601,7 @@ done: * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * - * 1. figure out a hyperslab (dimentions) and size (refer to get_hyperslab()). + * 1. figure out a hyperslab (dimensions) and size (refer to get_hyperslab()). * 2. Calculate the hyperslab selections as the selection is moving forward. * Selection would be same as the hyperslab except for the remaining edge portion * of the dataset. The code take care of the remaining portion if exist. @@ -643,6 +644,10 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti int req_filter; /* there was a request for a filter */ int req_obj_layout = 0; /* request layout to current object */ unsigned crt_order_flags; /* group creation order flag */ + H5_timer_t timer; /* Timer for read/write operations */ + H5_timevals_t times; /* Elapsed time for each operation */ + static double read_time = 0; + static double write_time = 0; h5tool_link_info_t linkinfo; unsigned i; unsigned u; @@ -661,7 +666,12 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose == 2) { + HDprintf("-----------------------------------------------------------------\n"); + HDprintf(" Type Filter (Compression) Timing read/write Name\n"); + HDprintf("-----------------------------------------------------------------\n"); + } + else { HDprintf("-----------------------------------------\n"); HDprintf(" Type Filter (Compression) Name\n"); HDprintf("-----------------------------------------\n"); @@ -682,7 +692,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ case H5TRAV_TYPE_GROUP: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "group", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "group", travt->objs[i].name); /* open input group */ @@ -749,6 +761,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti case H5TRAV_TYPE_DATASET: { hbool_t use_h5ocopy; + read_time = 0.0; + write_time = 0.0; + has_filter = 0; req_filter = 0; @@ -972,7 +987,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti H5P_DEFAULT, dcpl_out, H5P_DEFAULT); if (dset_out == H5I_INVALID_HID) { H5TOOLS_INFO("H5Dcreate2 failed"); - if (options->verbose) + if (options->verbose > 0) HDprintf(" warning: could not create dataset <%s>. Applying original " "settings\n", travt->objs[i].name); @@ -1013,11 +1028,27 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti } if (buf != NULL) { + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dread(dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dwrite(dset_out, wtype_id, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /* Check if we have VL data in the dataset's * datatype that must be reclaimed */ @@ -1115,12 +1146,28 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti hs_select_nelmts = 1; } /* end (else) rank == 0 */ + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dwrite(dset_out, wtype_id, hslab_space, f_space_id, dxpl_id, hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /* reclaim any VL memory, if necessary */ if (vl_data) @@ -1149,7 +1196,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti * print amount of compression used *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose > 0) { double ratio = 0; /* only print the compression ration if there was a filter request */ @@ -1160,10 +1207,12 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti /* compression ratio = uncompressed size / compressed size */ if (dsize_out != 0) ratio = (double)dsize_in / (double)dsize_out; - print_dataset_info(dcpl_out, travt->objs[i].name, ratio, 1); + print_dataset_info(dcpl_out, travt->objs[i].name, ratio, 1, options, + read_time, write_time); } else - print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0); + print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0, options, + read_time, write_time); /* print a message that the filter was not applied * (in case there was a filter) @@ -1220,6 +1269,10 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed"); + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Ocopy(fidin, /* Source file or group identifier */ travt->objs[i].name, /* Name of the source object to be copied */ fidout, /* Destination file or group identifier */ @@ -1227,6 +1280,11 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti ocpl_id, /* Properties which apply to the copy */ H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */ H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } if (H5Pclose(ocpl_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed"); @@ -1247,7 +1305,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (H5Dclose(dset_out) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, "dset", 0.0, write_time, travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); } /* end whether we have request for filter/chunking */ @@ -1260,7 +1320,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ case H5TRAV_TYPE_NAMED_DATATYPE: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "type", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "type", travt->objs[i].name); if ((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) @@ -1300,7 +1362,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti */ case H5TRAV_TYPE_LINK: case H5TRAV_TYPE_UDLINK: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "link", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); /* Check -X option. */ @@ -1425,7 +1489,8 @@ done: *------------------------------------------------------------------------- */ static void -print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) +print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr, pack_opt_t *options, double read_time, + double write_time) { char strfilter[255]; #if defined(PRINT_DEBUG) @@ -1514,7 +1579,10 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) } /* end for each filter */ if (!pr) - HDprintf(FORMAT_OBJ, "dset", objname); + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, "dset", read_time, write_time, objname); + else + HDprintf(FORMAT_OBJ, "dset", objname); else { char str[512], temp[512]; @@ -1522,7 +1590,10 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) HDstrcat(str, strfilter); HDsprintf(temp, " (%.3f:1)", ratio); HDstrcat(str, temp); - HDprintf(FORMAT_OBJ, str, objname); + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, str, read_time, write_time, objname); + else + HDprintf(FORMAT_OBJ, str, objname); } } /* end print_dataset_info() */ @@ -1550,7 +1621,7 @@ copy_user_block(const char *infile, const char *outfile, hsize_t size) /* Copy the userblock from the input file to the output file */ while (size > 0) { - ssize_t nread, nbytes; /* # of bytes transfered, etc. */ + ssize_t nread, nbytes; /* # of bytes transferred, etc. */ char rbuf[USERBLOCK_XFER_SIZE]; /* Buffer for reading */ const char *wbuf; /* Pointer into buffer, for writing */ diff --git a/tools/src/h5repack/h5repack_filters.c b/tools/src/h5repack/h5repack_filters.c index 32476ce..76bac5e 100644 --- a/tools/src/h5repack/h5repack_filters.c +++ b/tools/src/h5repack/h5repack_filters.c @@ -231,7 +231,7 @@ aux_assign_obj(const char * name, /* object name from traverse list */ * do extra checking in the case of SZIP; delete all filters in the case * of H5Z_FILTER_NONE present in the PACK_INFO_T filter array * - * Return: 0 success, -1 an error occured + * Return: 0 success, -1 an error occurred *------------------------------------------------------------------------- */ @@ -258,7 +258,7 @@ apply_filters(const char * name, /* object name from traverse list */ H5TOOLS_GOTO_DONE(0); /*------------------------------------------------------------------------- - * initialize the assigment object + * initialize the assignment object *------------------------------------------------------------------------- */ init_packobject(&obj); diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 3526268..4c88bd7 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -18,7 +18,7 @@ /* Name of tool */ #define PROGRAMNAME "h5repack" -static int parse_command_line(int argc, const char **argv, pack_opt_t *options); +static int parse_command_line(int argc, const char *const *argv, pack_opt_t *options); static void leave(int ret) H5_ATTR_NORETURN; /* module-scoped variables */ @@ -31,7 +31,7 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char * s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; +static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:EG:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:"; static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"block", require_arg, 'b'}, {"compact", require_arg, 'c'}, @@ -39,18 +39,18 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"file", require_arg, 'e'}, {"filter", require_arg, 'f'}, {"help", no_arg, 'h'}, - {"infile", require_arg, 'i'}, /* for backward compability */ + {"infile", require_arg, 'i'}, /* for backward compatibility */ {"low", require_arg, 'j'}, {"high", require_arg, 'k'}, {"layout", require_arg, 'l'}, {"minimum", require_arg, 'm'}, {"native", no_arg, 'n'}, - {"outfile", require_arg, 'o'}, /* for backward compability */ + {"outfile", require_arg, 'o'}, /* for backward compatibility */ {"sort_by", require_arg, 'q'}, {"ssize", require_arg, 's'}, {"threshold", require_arg, 't'}, {"ublock", require_arg, 'u'}, - {"verbose", no_arg, 'v'}, + {"verbose", optional_arg, 'v'}, {"sort_order", require_arg, 'z'}, {"enable-error-stack", no_arg, 'E'}, {"fs_pagesize", require_arg, 'G'}, @@ -68,6 +68,12 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"dst-vol-value", require_arg, '4'}, {"dst-vol-name", require_arg, '5'}, {"dst-vol-info", require_arg, '6'}, + {"src-vfd-value", require_arg, '7'}, + {"src-vfd-name", require_arg, '8'}, + {"src-vfd-info", require_arg, '9'}, + {"dst-vfd-value", require_arg, '0'}, + {"dst-vfd-name", require_arg, 'Y'}, + {"dst-vfd-info", require_arg, 'Z'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- @@ -88,7 +94,8 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " file2 Output HDF5 File\n"); PRINTVALSTREAM(rawoutstream, " OPTIONS\n"); PRINTVALSTREAM(rawoutstream, " -h, --help Print a usage message and exit\n"); - PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n"); + PRINTVALSTREAM(rawoutstream, " -v N, --verbose=N Verbose mode, print object information.\n"); + PRINTVALSTREAM(rawoutstream, " N - is an integer greater than 1, 2 displays read/write timing\n"); PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n"); PRINTVALSTREAM(rawoutstream, @@ -112,6 +119,24 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " --dst-vol-info VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the output HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " input HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-name Name of the VFL driver to use for opening the input\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the input HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " output HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-name Name of the VFL driver to use for opening the output\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the output HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n"); PRINTVALSTREAM(rawoutstream, " This option will take precedence over the options\n"); @@ -374,7 +399,7 @@ read_info(const char *filename, pack_opt_t *options) goto done; } - /* find begining of info */ + /* find beginning of info */ i = 0; c = '0'; while (c != ' ') { @@ -482,19 +507,25 @@ set_sort_order(const char *form) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char **argv, pack_opt_t *options) +parse_command_line(int argc, const char *const *argv, pack_opt_t *options) { h5tools_vol_info_t in_vol_info; h5tools_vol_info_t out_vol_info; - hbool_t custom_in_fapl = FALSE; - hbool_t custom_out_fapl = FALSE; - hid_t tmp_fapl = H5I_INVALID_HID; + h5tools_vfd_info_t in_vfd_info; + h5tools_vfd_info_t out_vfd_info; + hbool_t custom_in_vol = FALSE; + hbool_t custom_in_vfd = FALSE; + hbool_t custom_out_vol = FALSE; + hbool_t custom_out_vfd = FALSE; + hid_t tmp_fapl = H5I_INVALID_HID; int bound, opt; int ret_value = 0; /* Initialize fapl info structs */ HDmemset(&in_vol_info, 0, sizeof(h5tools_vol_info_t)); HDmemset(&out_vol_info, 0, sizeof(h5tools_vol_info_t)); + HDmemset(&in_vfd_info, 0, sizeof(h5tools_vfd_info_t)); + HDmemset(&out_vfd_info, 0, sizeof(h5tools_vfd_info_t)); /* parse command line options */ while (EOF != (opt = H5_get_option(argc, argv, s_opts, l_opts))) { @@ -525,7 +556,12 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) goto done; case 'v': - options->verbose = 1; + if (H5_optarg != NULL) { + if (2 == HDatoi(H5_optarg)) + options->verbose = 2; + } + else + options->verbose = 1; break; case 'f': @@ -665,7 +701,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 'a': options->alignment = HDstrtoull(H5_optarg, NULL, 0); if (options->alignment < 1) { - error_msg("invalid alignment size\n", H5_optarg); + error_msg("invalid alignment size `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -685,7 +721,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) else if (!HDstrcmp(strategy, "NONE")) options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { - error_msg("invalid file space management strategy\n", H5_optarg); + error_msg("invalid file space management strategy `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -741,13 +777,13 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case '1': in_vol_info.type = VOL_BY_VALUE; in_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); - custom_in_fapl = TRUE; + custom_in_vol = TRUE; break; case '2': in_vol_info.type = VOL_BY_NAME; in_vol_info.u.name = H5_optarg; - custom_in_fapl = TRUE; + custom_in_vol = TRUE; break; case '3': @@ -757,19 +793,51 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case '4': out_vol_info.type = VOL_BY_VALUE; out_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); - custom_out_fapl = TRUE; + custom_out_vol = TRUE; break; case '5': out_vol_info.type = VOL_BY_NAME; out_vol_info.u.name = H5_optarg; - custom_out_fapl = TRUE; + custom_out_vol = TRUE; break; case '6': out_vol_info.info_string = H5_optarg; break; + case '7': + in_vfd_info.type = VFD_BY_VALUE; + in_vfd_info.u.value = (H5FD_class_value_t)HDatoi(H5_optarg); + custom_in_vfd = TRUE; + break; + + case '8': + in_vfd_info.type = VFD_BY_NAME; + in_vfd_info.u.name = H5_optarg; + custom_in_vfd = TRUE; + break; + + case '9': + in_vfd_info.info = (const void *)H5_optarg; + break; + + case '0': + out_vfd_info.type = VFD_BY_VALUE; + out_vfd_info.u.value = (H5FD_class_value_t)HDatoi(H5_optarg); + custom_out_vfd = TRUE; + break; + + case 'Y': + out_vfd_info.type = VFD_BY_NAME; + out_vfd_info.u.name = H5_optarg; + custom_out_vfd = TRUE; + break; + + case 'Z': + out_vfd_info.info = (const void *)H5_optarg; + break; + default: break; } /* end switch */ @@ -803,8 +871,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) } /* Setup FAPL for input and output file accesses */ - if (custom_in_fapl) { - if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, &in_vol_info, NULL)) < 0) { + if (custom_in_vol || custom_in_vfd) { + if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL, + custom_in_vfd ? &in_vfd_info : NULL)) < 0) { error_msg("failed to setup FAPL for input file\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; @@ -823,8 +892,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) options->fin_fapl = tmp_fapl; } - if (custom_out_fapl) { - if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, &out_vol_info, NULL)) < 0) { + if (custom_out_vol || custom_out_vfd) { + if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, custom_out_vol ? &out_vol_info : NULL, + custom_out_vfd ? &out_vfd_info : NULL)) < 0) { error_msg("failed to setup FAPL for output file\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; @@ -858,7 +928,7 @@ done: *------------------------------------------------------------------------- */ int -main(int argc, const char **argv) +main(int argc, char **argv) { pack_opt_t options; /*the global options */ int parse_ret; @@ -888,7 +958,7 @@ main(int argc, const char **argv) /* Initialize default indexing options */ sort_by = H5_INDEX_CRT_ORDER; - parse_ret = parse_command_line(argc, argv, &options); + parse_ret = parse_command_line(argc, (const char *const *)argv, &options); if (parse_ret < 0) { HDprintf("Error occurred while parsing command-line options\n"); h5tools_setstatus(EXIT_FAILURE); diff --git a/tools/src/h5repack/h5repack_refs.c b/tools/src/h5repack/h5repack_refs.c index 6191624..3259b89 100644 --- a/tools/src/h5repack/h5repack_refs.c +++ b/tools/src/h5repack/h5repack_refs.c @@ -182,8 +182,11 @@ do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti /* create the reference, -1 parameter for objects */ if (H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, (hid_t)-1) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Rcreate failed"); - if (options->verbose) { - HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); + if (options->verbose > 0) { + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "dset", travt->objs[i].name); + else + HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); HDprintf("object <%s> object reference created to <%s>\n", travt->objs[i].name, refname); } @@ -276,7 +279,7 @@ do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti H5TOOLS_GOTO_ERROR((-1), "H5Rcreate failed"); if (H5Sclose(region_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Sclose failed"); - if (options->verbose) { + if (options->verbose > 0) { HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); HDprintf("object <%s> region reference created to <%s>\n", travt->objs[i].name, refname); @@ -536,7 +539,7 @@ copy_refs_attr(hid_t loc_in, hid_t loc_out, trav_table_t *travt, hid_t fidout) / } /* This line below needs to be moved in this loop instead of inserting outside. Otherwise, ref_comp_field_n may be >0 for the next attribute, which may not be - the reference type and will be accidently treated as the reference type. + the reference type and will be accidentally treated as the reference type. It will then cause the H5Acreate2 failed since that attribute is already created. KY 2020-02-07 */ diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c index 0542d03..71ee72e 100644 --- a/tools/src/h5repack/h5repack_verify.c +++ b/tools/src/h5repack/h5repack_verify.c @@ -378,11 +378,11 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t *------------------------------------------------------------------------- */ /* Open the files */ - if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, - (fname1_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, (fname1_fapl != H5P_DEFAULT), NULL, 0)) < + 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname1, H5FOPENERROR); - if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, - (fname2_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, (fname2_fapl != H5P_DEFAULT), NULL, 0)) < + 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname2, H5FOPENERROR); /*------------------------------------------------------------------------- @@ -397,7 +397,7 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t H5TOOLS_GOTO_ERROR((-1), "h5trav_gettable failed"); /*------------------------------------------------------------------------- - * traverse the suppplied object list + * traverse the supplied object list *------------------------------------------------------------------------- */ for (i = 0; i < trav->nobjs; i++) { |