diff options
Diffstat (limited to 'tools/h5repack/h5repack_copy.c')
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 604 |
1 files changed, 368 insertions, 236 deletions
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 03edd01..6b3e4d0 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -33,6 +33,8 @@ extern char *progname; #define FORMAT_OBJ_ATTR " %-21s %s\n" /* obj type, name */ +static +int do_hardlinks(hid_t fidout,trav_table_t *travt); /*------------------------------------------------------------------------- * Function: print_dataset_info @@ -148,12 +150,8 @@ static void print_dataset_info(hid_t dcpl_id, strcat(str,temp); printf(FORMAT_OBJ,str,objname); } - - - } - /*------------------------------------------------------------------------- * Function: copy_objects * @@ -209,11 +207,11 @@ int copy_objects(const char* fnamein, } /*------------------------------------------------------------------------- - * do the copy of referenced objects - * and create hard links + * create hard links *------------------------------------------------------------------------- */ - if(do_copy_refobjs(fidin,fidout,travt,options)<0) { + + if(do_hardlinks(fidout,travt)<0) { error_msg(progname, "<%s>: Could not copy data to: %s\n", fnamein, fnameout); goto out; } @@ -299,6 +297,8 @@ int do_copy_objects(hid_t fidin, void *buf=NULL; /* buffer for raw data */ void *sm_buf=NULL; /* buffer for raw data */ int i, j; + int nfilters; /* number of filters in DCPL */ + H5D_layout_t layout; /*------------------------------------------------------------------------- * copy the suppplied object list @@ -351,268 +351,337 @@ int do_copy_objects(hid_t fidin, *------------------------------------------------------------------------- */ case H5G_DATASET: - + +/*------------------------------------------------------------------------- + * early check for filters or a non default layout + *------------------------------------------------------------------------- + */ if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) goto error; - if ((f_space_id=H5Dget_space(dset_in))<0) - goto error; - if ((ftype_id=H5Dget_type (dset_in))<0) - goto error; if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) goto error; - if ((dcpl_out = H5Pcopy (dcpl_id))<0) - goto error; - if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) - goto error; - HDmemset(dims, 0, sizeof dims); - if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) - goto error; - nelmts=1; - for (j=0; j<rank; j++) - nelmts*=dims[j]; - - if (options->use_native==1) - wtype_id = h5tools_get_native_type(ftype_id); - else - wtype_id = H5Tcopy(ftype_id); - - if ((msize=H5Tget_size(wtype_id))==0) - goto error; - -/*------------------------------------------------------------------------- - * check for external files - *------------------------------------------------------------------------- - */ - if ((next=H5Pget_external_count (dcpl_id))<0) + if ((layout = H5Pget_layout(dcpl_id))<0) goto error; - if (next) - fprintf(stderr," <warning: %s has external files, ignoring read...>\n", - travt->objs[i].name ); - -/*------------------------------------------------------------------------- - * check if the dataset creation property list has filters that - * are not registered in the current configuration - * 1) the external filters GZIP and SZIP might not be available - * 2) the internal filters might be turned off - *------------------------------------------------------------------------- - */ - if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1) - { - apply_s=1; - apply_f=1; - + if (H5Dclose(dset_in)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + /*------------------------------------------------------------------------- - * references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - * in a second traversal of the output file + * check if we should use H5Ocopy or not + * if the DCPL has filters or a non default layout, we read/write the object + * otherwise we do a copy using H5Ocopy *------------------------------------------------------------------------- */ - if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) + + if (nfilters || layout!=H5D_CONTIGUOUS + /* does the dcpl has filters or non default layout ? */ + || + options->op_tbl->nelems + /* do we have input request for filter/chunking */ + || + options->all_filter==1 || options->all_layout==1 + ) { - /* get the storage size of the input dataset */ - dsize_in=H5Dget_storage_size(dset_in); - - /* check for datasets too small */ - if (nelmts*msize < options->threshold ) - apply_s=0; + if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) + goto error; + if ((f_space_id=H5Dget_space(dset_in))<0) + goto error; + if ((ftype_id=H5Dget_type (dset_in))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + if ((dcpl_out = H5Pcopy (dcpl_id))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) + goto error; + HDmemset(dims, 0, sizeof dims); + if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) + goto error; + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; - /* apply the filter */ - if (apply_s){ - if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) - goto error; - } + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - /*------------------------------------------------------------------------- - * create the output dataset; - * disable error checking in case the dataset cannot be created with the - * modified dcpl; in that case use the original instead - *------------------------------------------------------------------------- + if ((msize=H5Tget_size(wtype_id))==0) + goto error; + + /*------------------------------------------------------------------------- + * check for external files + *------------------------------------------------------------------------- + */ + if ((next=H5Pget_external_count (dcpl_id))<0) + goto error; + + if (next) + fprintf(stderr," <warning: %s has external files, ignoring read...>\n", + travt->objs[i].name ); + + /*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- */ - H5E_BEGIN_TRY { - dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); - } H5E_END_TRY; - if (dset_out==FAIL) + if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1) { - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) - goto error; - apply_f=0; - } - - /*------------------------------------------------------------------------- - * read/write - *------------------------------------------------------------------------- - */ - if (nelmts) - { - size_t need = (size_t)(nelmts*msize); /* bytes needed */ - if ( need < H5TOOLS_MALLOCSIZE ) - buf = HDmalloc(need); - - if (buf != NULL ) - { - if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - } + apply_s=1; + apply_f=1; - else /* possibly not enough memory, read/write by hyperslabs */ - + /*------------------------------------------------------------------------- + * references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * in a second traversal of the output file + *------------------------------------------------------------------------- + */ + if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) { - size_t p_type_nbytes = msize; /*size of memory type */ - hsize_t p_nelmts = nelmts; /*total selected elmts */ - hsize_t elmtno; /*counter */ - int carry; /*counter carry value */ - unsigned int vl_data = 0; /*contains VL datatypes */ + /* get the storage size of the input dataset */ + dsize_in=H5Dget_storage_size(dset_in); - /* stripmine info */ - hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ - hsize_t sm_nbytes; /*bytes per stripmine */ - hsize_t sm_nelmts; /*elements per stripmine*/ - hid_t sm_space; /*stripmine data space */ + /* check for datasets too small */ + if (nelmts*msize < options->threshold ) + apply_s=0; - /* hyperslab info */ - hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ - hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ - hsize_t hs_nelmts; /*elements in request */ - hsize_t zero[8]; /*vector of zeros */ + /* apply the filter */ + if (apply_s){ + if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + goto error; + } - /* check if we have VL data in the dataset's datatype */ - if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) - vl_data = TRUE; - - /* - * determine the strip mine size and allocate a buffer. The strip mine is - * a hyperslab whose size is manageable. + /*------------------------------------------------------------------------- + * create the output dataset; + * disable error checking in case the dataset cannot be created with the + * modified dcpl; in that case use the original instead + *------------------------------------------------------------------------- */ - sm_nbytes = p_type_nbytes; - - for (i = rank; i > 0; --i) { - sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); - sm_nbytes *= sm_size[i - 1]; - assert(sm_nbytes > 0); + H5E_BEGIN_TRY { + dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); + } H5E_END_TRY; + if (dset_out==FAIL) + { + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) + goto error; + apply_f=0; } - sm_buf = HDmalloc((size_t)sm_nbytes); - - sm_nelmts = sm_nbytes / p_type_nbytes; - sm_space = H5Screate_simple(1, &sm_nelmts, NULL); - /* the stripmine loop */ - memset(hs_offset, 0, sizeof hs_offset); - memset(zero, 0, sizeof zero); - - for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + /*------------------------------------------------------------------------- + * read/write + *------------------------------------------------------------------------- + */ + if (nelmts) { - /* calculate the hyperslab size */ - if (rank > 0) + size_t need = (size_t)(nelmts*msize); /* bytes needed */ + if ( need < H5TOOLS_MALLOCSIZE ) + buf = HDmalloc(need); + + if (buf != NULL ) { - for (i = 0, hs_nelmts = 1; i < rank; i++) - { - hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); - hs_nelmts *= hs_size[i]; - } - - if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - } - else - { - H5Sselect_all(f_space_id); - H5Sselect_all(sm_space); - hs_nelmts = 1; - } /* rank */ - - /* read/write */ - if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; - if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; + } - /* reclaim any VL memory, if necessary */ - if(vl_data) - H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); - - /* calculate the next hyperslab offset */ - for (i = rank, carry = 1; i > 0 && carry; --i) + else /* possibly not enough memory, read/write by hyperslabs */ + { - hs_offset[i - 1] += hs_size[i - 1]; - if (hs_offset[i - 1] == dims[i - 1]) - hs_offset[i - 1] = 0; - else - carry = 0; - } /* i */ - } /* elmtno */ - - H5Sclose(sm_space); - /* free */ - if (sm_buf!=NULL) - { - HDfree(sm_buf); - sm_buf=NULL; - } - } /* hyperslab read */ + size_t p_type_nbytes = msize; /*size of memory type */ + hsize_t p_nelmts = nelmts; /*total selected elmts */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + hsize_t sm_nelmts; /*elements per stripmine*/ + hid_t sm_space; /*stripmine data space */ + + /* hyperslab info */ + hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ + hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ + hsize_t hs_nelmts; /*elements in request */ + hsize_t zero[8]; /*vector of zeros */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* + * determine the strip mine size and allocate a buffer. The strip mine is + * a hyperslab whose size is manageable. + */ + sm_nbytes = p_type_nbytes; + + for (i = rank; i > 0; --i) { + sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); + sm_nbytes *= sm_size[i - 1]; + assert(sm_nbytes > 0); + } + sm_buf = HDmalloc((size_t)sm_nbytes); + + sm_nelmts = sm_nbytes / p_type_nbytes; + sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + + /* the stripmine loop */ + memset(hs_offset, 0, sizeof hs_offset); + memset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + { + /* calculate the hyperslab size */ + if (rank > 0) + { + for (i = 0, hs_nelmts = 1; i < rank; i++) + { + hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); + hs_nelmts *= hs_size[i]; + } + + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + goto error; + } + else + { + H5Sselect_all(f_space_id); + H5Sselect_all(sm_space); + hs_nelmts = 1; + } /* rank */ + + /* read/write */ + if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + + /* reclaim any VL memory, if necessary */ + if(vl_data) + H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); + + /* calculate the next hyperslab offset */ + for (i = rank, carry = 1; i > 0 && carry; --i) + { + hs_offset[i - 1] += hs_size[i - 1]; + if (hs_offset[i - 1] == dims[i - 1]) + hs_offset[i - 1] = 0; + else + carry = 0; + } /* i */ + } /* elmtno */ + + H5Sclose(sm_space); + /* free */ + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } + } /* hyperslab read */ }/*nelmts*/ - - /*------------------------------------------------------------------------- - * amount of compression used - *------------------------------------------------------------------------- - */ - if (options->verbose) - { - if (apply_s && apply_f) + + /*------------------------------------------------------------------------- + * amount of compression used + *------------------------------------------------------------------------- + */ + if (options->verbose) { - /* get the storage size of the input dataset */ - dsize_out=H5Dget_storage_size(dset_out); - PER((hssize_t)dsize_in,(hssize_t)dsize_out); - print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0); + if (apply_s && apply_f) + { + /* get the storage size of the input dataset */ + dsize_out=H5Dget_storage_size(dset_out); + PER((hssize_t)dsize_in,(hssize_t)dsize_out); + print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0); + } + else + print_dataset_info(dcpl_id,travt->objs[i].name,0); } - else - print_dataset_info(dcpl_id,travt->objs[i].name,0); - } - - if (apply_s==0 && options->verbose) - printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", - travt->objs[i].name, - (int)options->threshold); - - if (apply_f==0 && options->verbose) + + if (apply_s==0 && options->verbose) + printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", + travt->objs[i].name, + (int)options->threshold); + + if (apply_f==0 && options->verbose) printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); - - /*------------------------------------------------------------------------- - * copy attrs - *------------------------------------------------------------------------- - */ - if (copy_attr(dset_in,dset_out,options)<0) - goto error; - - /*close */ - if (H5Dclose(dset_out)<0) - goto error; - + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(dset_in,dset_out,options)<0) + goto error; + + /*close */ + if (H5Dclose(dset_out)<0) + goto error; + }/*H5T_STD_REF_OBJ*/ }/*can_read*/ + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Tclose(ftype_id)<0) + goto error; + if (H5Tclose(wtype_id)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Pclose(dcpl_out)<0) + goto error; + if (H5Sclose(f_space_id)<0) + goto error; + if (H5Dclose(dset_in)<0) + goto error; + } + /*------------------------------------------------------------------------- + * we do not have request for filter/chunking use H5Ocopy instead + *------------------------------------------------------------------------- + */ + else + { + hid_t pid; + + /* create property to pass copy options */ + if ( (pid = H5Pcreate(H5P_OBJECT_COPY)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * do the copy + *------------------------------------------------------------------------- + */ + + if (H5Ocopy(fidin, /* Source file or group identifier */ + travt->objs[i].name, /* Name of the source object to be copied */ + fidout, /* Destination file or group identifier */ + travt->objs[i].name, /* Name of the destination object */ + pid, /* Properties which apply to the copy */ + H5P_DEFAULT)<0) /* Properties which apply to the new hard link */ + goto error; + + /* close property */ + if (H5Pclose(pid)<0) + goto error; + + } /* end do we have request for filter/chunking */ -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - if (H5Tclose(ftype_id)<0) - goto error; - if (H5Tclose(wtype_id)<0) - goto error; - if (H5Pclose(dcpl_id)<0) - goto error; - if (H5Pclose(dcpl_out)<0) - goto error; - if (H5Sclose(f_space_id)<0) - goto error; - if (H5Dclose(dset_in)<0) - goto error; break; @@ -686,8 +755,6 @@ int do_copy_objects(hid_t fidin, /*------------------------------------------------------------------------- * the root is a special case, we get an ID for the root group * and copy its attributes using that ID - * it must be done last, because the attributes might contain references to - * objects in the object list *------------------------------------------------------------------------- */ @@ -827,8 +894,6 @@ int copy_attr(hid_t loc_in, */ if ( ! H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { - - /*------------------------------------------------------------------------- * read to memory *------------------------------------------------------------------------- @@ -894,6 +959,73 @@ error: return -1; } +/*------------------------------------------------------------------------- + * Function: do_hardlinks + * + * Purpose: duplicate hard links + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December, 10, 2003 + * + *------------------------------------------------------------------------- + */ + +static +int do_hardlinks(hid_t fidout,trav_table_t *travt) +{ + int i, j; + + for ( i = 0; i < travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + case H5G_GROUP: + + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++) + { + if (H5Glink(fidout, + H5L_TYPE_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name)<0) + return -1; + } + } + + break; + + case H5G_DATASET: + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++){ + if (H5Glink(fidout, + H5L_TYPE_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name)<0) + return -1; + } + } + break; + case H5G_TYPE: + case H5G_LINK: + case H5G_UDLINK: + + /*nothing to do */ + break; + + default: + + break; + } + } + + return 0; + +} |