diff options
Diffstat (limited to 'tools/src/h5repack/h5repack_copy.c')
-rw-r--r-- | tools/src/h5repack/h5repack_copy.c | 198 |
1 files changed, 137 insertions, 61 deletions
diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 0567269..c638f3d 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -80,8 +80,8 @@ static herr_t walk_error_callback(H5_ATTR_UNUSED unsigned n, const H5E_error2_t * -1 no *------------------------------------------------------------------------- */ - -int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) +int +copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) { int ret_value = 0; hid_t fidin = -1; @@ -387,7 +387,7 @@ done: trav_table_free(travt); return ret_value; -} +} /* end copy_objects() */ /*------------------------------------------------------------------------- * Function: Get_hyperslab @@ -543,7 +543,7 @@ Get_hyperslab(hid_t dcpl_id, int rank_dset, hsize_t dims_dset[], done: return ret_value; -} +} /* end Get_hyperslab() */ /*------------------------------------------------------------------------- * Function: do_copy_objects @@ -599,7 +599,8 @@ done: *------------------------------------------------------------------------- */ -int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, +int +do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *options) /* repack options */ { int ret_value = 0; @@ -699,7 +700,11 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, } else { if (options->grp_compact > 0 || options->grp_indexed > 0) - if (H5Pset_link_phase_change(gcpl_out, (unsigned) options->grp_compact, (unsigned) options->grp_indexed) < 0) + if (H5Pset_link_phase_change( + gcpl_out, + (unsigned) options->grp_compact, + (unsigned) options->grp_indexed) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_phase_change failed"); if ((grp_out = H5Gcreate2(fidout, travt->objs[i].name, H5P_DEFAULT, gcpl_out, H5P_DEFAULT)) < 0) @@ -780,6 +785,9 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (H5Dclose(dset_in) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); +/* TODO: be smart about external storage */ +/* can we use the dcpl_in/out to adjust any external storage? */ +/* what about the case with just H5Ocopy? */ /*------------------------------------------------------------------------- * check if we should use H5Ocopy or not * if there is a request for filters/layout, we read/write the object @@ -799,8 +807,17 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); if ((dcpl_in = H5Dget_create_plist(dset_in)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); - if ((dcpl_out = H5Pcopy(dcpl_in)) < 0) - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed"); + /* If the input dataset has external storage, it must be contiguous. + * Accordingly, there would be no filter or chunk properties to preserve. + */ + if (H5Pget_external_count(dcpl_in)) { + if ((dcpl_out = H5Pcreate(H5P_DATASET_CREATE)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed"); + } + else { + if ((dcpl_out = H5Pcopy(dcpl_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed"); + } if ((rank = H5Sget_simple_extent_ndims(f_space_id)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed"); HDmemset(dims, 0, sizeof dims); @@ -819,7 +836,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, wtype_id = H5Tget_native_type(ftype_id, H5T_DIR_DEFAULT); else wtype_id = H5Tcopy(ftype_id); - } /* end if */ + } if ((msize = H5Tget_size(wtype_id)) == 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); @@ -858,12 +875,20 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, /* apply the filter */ if (apply_s) - if (apply_filters(travt->objs[i].name, rank, dims, msize, dcpl_out, options, &has_filter) < 0) + if (apply_filters( + travt->objs[i].name, + rank, + dims, + msize, + dcpl_out, + options, + &has_filter) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "apply_filters failed"); /* only if layout change requested for entire file or * individual obj */ - if (options->all_layout > 0 || req_obj_layout == 1) + if (options->all_layout > 0 || req_obj_layout == 1) { /*------------------------------------------------- * Unset the unlimited max dims if convert to other * than chunk layouts, because unlimited max dims @@ -883,14 +908,14 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, /* if convert to COMPACT */ if (options->layout_g == H5D_COMPACT) - /* should be smaller than 64K */ if (size_dset > MAX_COMPACT_DSIZE) limit_maxdims = FALSE; /* unset unlimited max dims */ if (limit_maxdims) H5Sset_extent_simple(f_space_id, rank, dims, NULL); - } + } /* if not chunked */ + } /* if layout change requested for entire file or individual object */ /*------------------------------------------------------------------------- * create the output dataset; @@ -898,16 +923,36 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, * modified dcpl; in that case use the original instead *------------------------------------------------------------------------- */ - dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT); - if (dset_out == FAIL) { - H5Epush2(H5tools_ERR_STACK_g, __FILE__, FUNC, __LINE__, H5tools_ERR_CLS_g, H5E_tools_g, H5E_tools_min_id_g, "H5Dcreate2 failed"); + dset_out = H5Dcreate2(fidout, + travt->objs[i].name, + wtype_id, + f_space_id, + H5P_DEFAULT, + dcpl_out, + H5P_DEFAULT); + if (dset_out < 0) { + H5Epush2(H5tools_ERR_STACK_g, + __FILE__, + FUNC, + __LINE__, + H5tools_ERR_CLS_g, + H5E_tools_g, + H5E_tools_min_id_g, + "H5Dcreate2 failed"); if (options->verbose) - printf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name); - - if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) + printf(" warning: could not create dataset <%s>. Applying original settings\n", + travt->objs[i].name); + dset_out = H5Dcreate2(fidout, + travt->objs[i].name, + wtype_id, + f_space_id, + H5P_DEFAULT, + dcpl_in, + H5P_DEFAULT); + if (dset_out < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed"); apply_f = 0; - } + } /* if opening dataset with original DCPL (retry) */ /*------------------------------------------------------------------------- * read/write @@ -931,8 +976,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (TRUE == H5Tdetect_class(wtype_id, H5T_VLEN)) if (H5Dvlen_reclaim(wtype_id, f_space_id, H5P_DEFAULT, buf) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dvlen_reclaim failed"); - /* free */ - if (buf != NULL) { + + if (buf != NULL) { /* TODO: is buf potentially released by H5Dvlen_reclaim()? */ HDfree(buf); buf = NULL; } @@ -968,17 +1013,25 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, dset_layout = H5Pget_layout(dcpl_out); if (dset_layout == H5D_CHUNKED) dcpl_tmp = dcpl_out; /* writing dataset */ - else { /* if reading dataset is chunked */ + else { dset_layout = H5Pget_layout(dcpl_in); if (dset_layout == H5D_CHUNKED) dcpl_tmp = dcpl_in; /* reading dataset */ } - /* get hyperslab dims and size in byte */ - if (Get_hyperslab(dcpl_tmp, rank, dims, p_type_nbytes, hslab_dims, &hslab_nbytes) < 0) + if (Get_hyperslab( + dcpl_tmp, + rank, + dims, + p_type_nbytes, + hslab_dims, + &hslab_nbytes) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Get_hyperslab failed"); hslab_buf = HDmalloc((size_t)hslab_nbytes); + if (hslab_buf == NULL) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "can't allocate space for hyperslab"); hslab_nelmts = hslab_nbytes / p_type_nbytes; hslab_space = H5Screate_simple(1, &hslab_nelmts, NULL); @@ -996,28 +1049,57 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, */ for (k = 0, hs_select_nelmts = 1; k < rank; k++) { /* MIN() is used to get the remaining edge portion if exist. - * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is smaller then the hyperslab.*/ + * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is + * smaller then the hyperslab. + */ hs_sel_count[k] = MIN(dims[k] - hs_sel_offset[k], hslab_dims[k]); hs_select_nelmts *= hs_sel_count[k]; } - if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_sel_offset, NULL, hs_sel_count, NULL) < 0) + if (H5Sselect_hyperslab( + f_space_id, + H5S_SELECT_SET, + hs_sel_offset, + NULL, + hs_sel_count, + NULL) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed"); - if (H5Sselect_hyperslab(hslab_space, H5S_SELECT_SET, zero, NULL, &hs_select_nelmts, NULL) < 0) + if (H5Sselect_hyperslab( + hslab_space, + H5S_SELECT_SET, + zero, + NULL, + &hs_select_nelmts, + NULL) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed"); - } + } /* end if rank > 0 */ else { H5Sselect_all(f_space_id); H5Sselect_all(hslab_space); hs_select_nelmts = 1; - } /* rank */ - - if(H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0) + } /* rank == 0 */ + + if (H5Dread( + dset_in, + wtype_id, + hslab_space, + f_space_id, + H5P_DEFAULT, + hslab_buf) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dread failed"); - if(H5Dwrite(dset_out, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0) + if (H5Dwrite( + dset_out, + wtype_id, + hslab_space, + f_space_id, + H5P_DEFAULT, + hslab_buf) + < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dwrite failed"); - /* reclaim any VL memory, if necessary */ if (vl_data) H5Dvlen_reclaim(wtype_id, hslab_space, H5P_DEFAULT, hslab_buf); @@ -1029,20 +1111,19 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, hs_sel_offset[k - 1] = 0; else carry = 0; - } /* k */ - } /* elmtno */ + } + } /* hyperslab selection loop */ H5Sclose(hslab_space); - /* free */ if (hslab_buf != NULL) { HDfree(hslab_buf); hslab_buf = NULL; } - } /* hyperslab read */ - } /* if (nelmts>0 && space_status==H5D_SPACE_STATUS_NOT_ALLOCATED) */ + } /* end if reading/writing by hyperslab */ + } /* if (nelmts > 0 && space_status != H5D_SPACE_STATUS_NOT_ALLOCATED) */ /*------------------------------------------------------------------------- - * amount of compression used + * print amount of compression used *------------------------------------------------------------------------- */ if (options->verbose) { @@ -1062,14 +1143,16 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0); /* print a message that the filter was not applied - (in case there was a filter) + * (in case there was a filter) */ if (has_filter && apply_s == 0) - printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp); + printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", + travt->objs[i].name, + (int) options->min_comp); if (has_filter && apply_f == 0) printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); - } /* verbose */ + } /* if verbose (print compression) */ /*------------------------------------------------------------------------- * copy attrs @@ -1078,11 +1161,10 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); - /*close */ if (H5Dclose(dset_out) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); - }/*!H5T_REFERENCE*/ - }/*h5tools_canreadf*/ + } /* !H5T_REFERENCE */ + } /* if h5tools_canreadf (filter availability check) */ /*------------------------------------------------------------------------- * close @@ -1102,7 +1184,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); } /*------------------------------------------------------------------------- - * we do not have request for filter/chunking use H5Ocopy instead + * we do not have request for filter/chunking; use H5Ocopy instead *------------------------------------------------------------------------- */ else { @@ -1116,11 +1198,6 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_copy_object failed"); - /*------------------------------------------------------------------------- - * do the copy - *------------------------------------------------------------------------- - */ - if (H5Ocopy(fidin, /* Source file or group identifier */ travt->objs[i].name, /* Name of the source object to be copied */ fidout, /* Destination file or group identifier */ @@ -1129,7 +1206,6 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Ocopy failed"); - /* close property */ if (H5Pclose(pid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); @@ -1151,7 +1227,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (options->verbose) printf(FORMAT_OBJ, "dset", travt->objs[i].name); - } /* end do we have request for filter/chunking */ + } /* end whether we have request for filter/chunking */ break; /*------------------------------------------------------------------------- @@ -1211,8 +1287,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, default: HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Object type not found"); } /* switch */ - } /* end for */ - } /* end if */ + } /* end for each object to traverse */ + } /* end if there are objects */ done: @@ -1250,7 +1326,7 @@ done: HDfree(hslab_buf); return ret_value; -} +} /* end do_copy_objects() */ /*------------------------------------------------------------------------- * Function: print_dataset_info @@ -1345,7 +1421,7 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) HDstrcat(strfilter, "UD "); break; } /* switch */ - }/*i*/ + } /* for each filter */ if (!pr) printf(FORMAT_OBJ, "dset", objname); @@ -1358,7 +1434,7 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) HDstrcat(str, temp); printf(FORMAT_OBJ, str, objname); } -} +} /* end print_dataset_info() */ /*------------------------------------------------------------------------- * Function: copy_user_block @@ -1428,7 +1504,7 @@ done: HDclose(outfid); return ret_value; -} +} /* end copy_user_block() */ /*------------------------------------------------------------------------- * Function: print_user_block @@ -1501,6 +1577,6 @@ done: HDclose(fh); return; -} +} /* end print_user_block() */ #endif |