diff options
| -rw-r--r-- | tools/h5repack/h5repack.h | 8 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_copy.c | 604 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_main.c | 4 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_refs.c | 720 |
4 files changed, 369 insertions, 967 deletions
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index da82a05..0f5ccfe 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -102,6 +102,7 @@ typedef struct { int verbose; /*verbose mode */ hsize_t threshold; /*minimum size to compress, in bytes */ int use_native; /*use a native type in write */ + int have_fc_f; /*has input to use filter/chunking flag */ } pack_opt_t; @@ -158,13 +159,6 @@ int copy_attr(hid_t loc_in, pack_opt_t *options ); -int do_copy_refobjs(hid_t fidin, - hid_t fidout, - trav_table_t *travt, - pack_opt_t *options); /* repack options */ - - - void read_info(const char *filename,pack_opt_t *options); void init_packobject(pack_info_t *obj); int print_filters(hid_t dcpl_id); diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 03edd01..6b3e4d0 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -33,6 +33,8 @@ extern char *progname; #define FORMAT_OBJ_ATTR " %-21s %s\n" /* obj type, name */ +static +int do_hardlinks(hid_t fidout,trav_table_t *travt); /*------------------------------------------------------------------------- * Function: print_dataset_info @@ -148,12 +150,8 @@ static void print_dataset_info(hid_t dcpl_id, strcat(str,temp); printf(FORMAT_OBJ,str,objname); } - - - } - /*------------------------------------------------------------------------- * Function: copy_objects * @@ -209,11 +207,11 @@ int copy_objects(const char* fnamein, } /*------------------------------------------------------------------------- - * do the copy of referenced objects - * and create hard links + * create hard links *------------------------------------------------------------------------- */ - if(do_copy_refobjs(fidin,fidout,travt,options)<0) { + + if(do_hardlinks(fidout,travt)<0) { error_msg(progname, "<%s>: Could not copy data to: %s\n", fnamein, fnameout); goto out; } @@ -299,6 +297,8 @@ int do_copy_objects(hid_t fidin, void *buf=NULL; /* buffer for raw data */ void *sm_buf=NULL; /* buffer for raw data */ int i, j; + int nfilters; /* number of filters in DCPL */ + H5D_layout_t layout; /*------------------------------------------------------------------------- * copy the suppplied object list @@ -351,268 +351,337 @@ int do_copy_objects(hid_t fidin, *------------------------------------------------------------------------- */ case H5G_DATASET: - + +/*------------------------------------------------------------------------- + * early check for filters or a non default layout + *------------------------------------------------------------------------- + */ if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) goto error; - if ((f_space_id=H5Dget_space(dset_in))<0) - goto error; - if ((ftype_id=H5Dget_type (dset_in))<0) - goto error; if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) goto error; - if ((dcpl_out = H5Pcopy (dcpl_id))<0) - goto error; - if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) - goto error; - HDmemset(dims, 0, sizeof dims); - if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) - goto error; - nelmts=1; - for (j=0; j<rank; j++) - nelmts*=dims[j]; - - if (options->use_native==1) - wtype_id = h5tools_get_native_type(ftype_id); - else - wtype_id = H5Tcopy(ftype_id); - - if ((msize=H5Tget_size(wtype_id))==0) - goto error; - -/*------------------------------------------------------------------------- - * check for external files - *------------------------------------------------------------------------- - */ - if ((next=H5Pget_external_count (dcpl_id))<0) + if ((layout = H5Pget_layout(dcpl_id))<0) goto error; - if (next) - fprintf(stderr," <warning: %s has external files, ignoring read...>\n", - travt->objs[i].name ); - -/*------------------------------------------------------------------------- - * check if the dataset creation property list has filters that - * are not registered in the current configuration - * 1) the external filters GZIP and SZIP might not be available - * 2) the internal filters might be turned off - *------------------------------------------------------------------------- - */ - if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1) - { - apply_s=1; - apply_f=1; - + if (H5Dclose(dset_in)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + /*------------------------------------------------------------------------- - * references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - * in a second traversal of the output file + * check if we should use H5Ocopy or not + * if the DCPL has filters or a non default layout, we read/write the object + * otherwise we do a copy using H5Ocopy *------------------------------------------------------------------------- */ - if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) + + if (nfilters || layout!=H5D_CONTIGUOUS + /* does the dcpl has filters or non default layout ? */ + || + options->op_tbl->nelems + /* do we have input request for filter/chunking */ + || + options->all_filter==1 || options->all_layout==1 + ) { - /* get the storage size of the input dataset */ - dsize_in=H5Dget_storage_size(dset_in); - - /* check for datasets too small */ - if (nelmts*msize < options->threshold ) - apply_s=0; + if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) + goto error; + if ((f_space_id=H5Dget_space(dset_in))<0) + goto error; + if ((ftype_id=H5Dget_type (dset_in))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + if ((dcpl_out = H5Pcopy (dcpl_id))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) + goto error; + HDmemset(dims, 0, sizeof dims); + if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) + goto error; + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; - /* apply the filter */ - if (apply_s){ - if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) - goto error; - } + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - /*------------------------------------------------------------------------- - * create the output dataset; - * disable error checking in case the dataset cannot be created with the - * modified dcpl; in that case use the original instead - *------------------------------------------------------------------------- + if ((msize=H5Tget_size(wtype_id))==0) + goto error; + + /*------------------------------------------------------------------------- + * check for external files + *------------------------------------------------------------------------- + */ + if ((next=H5Pget_external_count (dcpl_id))<0) + goto error; + + if (next) + fprintf(stderr," <warning: %s has external files, ignoring read...>\n", + travt->objs[i].name ); + + /*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- */ - H5E_BEGIN_TRY { - dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); - } H5E_END_TRY; - if (dset_out==FAIL) + if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1) { - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) - goto error; - apply_f=0; - } - - /*------------------------------------------------------------------------- - * read/write - *------------------------------------------------------------------------- - */ - if (nelmts) - { - size_t need = (size_t)(nelmts*msize); /* bytes needed */ - if ( need < H5TOOLS_MALLOCSIZE ) - buf = HDmalloc(need); - - if (buf != NULL ) - { - if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - } + apply_s=1; + apply_f=1; - else /* possibly not enough memory, read/write by hyperslabs */ - + /*------------------------------------------------------------------------- + * references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * in a second traversal of the output file + *------------------------------------------------------------------------- + */ + if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) { - size_t p_type_nbytes = msize; /*size of memory type */ - hsize_t p_nelmts = nelmts; /*total selected elmts */ - hsize_t elmtno; /*counter */ - int carry; /*counter carry value */ - unsigned int vl_data = 0; /*contains VL datatypes */ + /* get the storage size of the input dataset */ + dsize_in=H5Dget_storage_size(dset_in); - /* stripmine info */ - hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ - hsize_t sm_nbytes; /*bytes per stripmine */ - hsize_t sm_nelmts; /*elements per stripmine*/ - hid_t sm_space; /*stripmine data space */ + /* check for datasets too small */ + if (nelmts*msize < options->threshold ) + apply_s=0; - /* hyperslab info */ - hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ - hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ - hsize_t hs_nelmts; /*elements in request */ - hsize_t zero[8]; /*vector of zeros */ + /* apply the filter */ + if (apply_s){ + if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + goto error; + } - /* check if we have VL data in the dataset's datatype */ - if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) - vl_data = TRUE; - - /* - * determine the strip mine size and allocate a buffer. The strip mine is - * a hyperslab whose size is manageable. + /*------------------------------------------------------------------------- + * create the output dataset; + * disable error checking in case the dataset cannot be created with the + * modified dcpl; in that case use the original instead + *------------------------------------------------------------------------- */ - sm_nbytes = p_type_nbytes; - - for (i = rank; i > 0; --i) { - sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); - sm_nbytes *= sm_size[i - 1]; - assert(sm_nbytes > 0); + H5E_BEGIN_TRY { + dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); + } H5E_END_TRY; + if (dset_out==FAIL) + { + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) + goto error; + apply_f=0; } - sm_buf = HDmalloc((size_t)sm_nbytes); - - sm_nelmts = sm_nbytes / p_type_nbytes; - sm_space = H5Screate_simple(1, &sm_nelmts, NULL); - /* the stripmine loop */ - memset(hs_offset, 0, sizeof hs_offset); - memset(zero, 0, sizeof zero); - - for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + /*------------------------------------------------------------------------- + * read/write + *------------------------------------------------------------------------- + */ + if (nelmts) { - /* calculate the hyperslab size */ - if (rank > 0) + size_t need = (size_t)(nelmts*msize); /* bytes needed */ + if ( need < H5TOOLS_MALLOCSIZE ) + buf = HDmalloc(need); + + if (buf != NULL ) { - for (i = 0, hs_nelmts = 1; i < rank; i++) - { - hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); - hs_nelmts *= hs_size[i]; - } - - if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - } - else - { - H5Sselect_all(f_space_id); - H5Sselect_all(sm_space); - hs_nelmts = 1; - } /* rank */ - - /* read/write */ - if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; - if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; + } - /* reclaim any VL memory, if necessary */ - if(vl_data) - H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); - - /* calculate the next hyperslab offset */ - for (i = rank, carry = 1; i > 0 && carry; --i) + else /* possibly not enough memory, read/write by hyperslabs */ + { - hs_offset[i - 1] += hs_size[i - 1]; - if (hs_offset[i - 1] == dims[i - 1]) - hs_offset[i - 1] = 0; - else - carry = 0; - } /* i */ - } /* elmtno */ - - H5Sclose(sm_space); - /* free */ - if (sm_buf!=NULL) - { - HDfree(sm_buf); - sm_buf=NULL; - } - } /* hyperslab read */ + size_t p_type_nbytes = msize; /*size of memory type */ + hsize_t p_nelmts = nelmts; /*total selected elmts */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + hsize_t sm_nelmts; /*elements per stripmine*/ + hid_t sm_space; /*stripmine data space */ + + /* hyperslab info */ + hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ + hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ + hsize_t hs_nelmts; /*elements in request */ + hsize_t zero[8]; /*vector of zeros */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* + * determine the strip mine size and allocate a buffer. The strip mine is + * a hyperslab whose size is manageable. + */ + sm_nbytes = p_type_nbytes; + + for (i = rank; i > 0; --i) { + sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); + sm_nbytes *= sm_size[i - 1]; + assert(sm_nbytes > 0); + } + sm_buf = HDmalloc((size_t)sm_nbytes); + + sm_nelmts = sm_nbytes / p_type_nbytes; + sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + + /* the stripmine loop */ + memset(hs_offset, 0, sizeof hs_offset); + memset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + { + /* calculate the hyperslab size */ + if (rank > 0) + { + for (i = 0, hs_nelmts = 1; i < rank; i++) + { + hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); + hs_nelmts *= hs_size[i]; + } + + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + goto error; + } + else + { + H5Sselect_all(f_space_id); + H5Sselect_all(sm_space); + hs_nelmts = 1; + } /* rank */ + + /* read/write */ + if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + + /* reclaim any VL memory, if necessary */ + if(vl_data) + H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); + + /* calculate the next hyperslab offset */ + for (i = rank, carry = 1; i > 0 && carry; --i) + { + hs_offset[i - 1] += hs_size[i - 1]; + if (hs_offset[i - 1] == dims[i - 1]) + hs_offset[i - 1] = 0; + else + carry = 0; + } /* i */ + } /* elmtno */ + + H5Sclose(sm_space); + /* free */ + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } + } /* hyperslab read */ }/*nelmts*/ - - /*------------------------------------------------------------------------- - * amount of compression used - *------------------------------------------------------------------------- - */ - if (options->verbose) - { - if (apply_s && apply_f) + + /*------------------------------------------------------------------------- + * amount of compression used + *------------------------------------------------------------------------- + */ + if (options->verbose) { - /* get the storage size of the input dataset */ - dsize_out=H5Dget_storage_size(dset_out); - PER((hssize_t)dsize_in,(hssize_t)dsize_out); - print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0); + if (apply_s && apply_f) + { + /* get the storage size of the input dataset */ + dsize_out=H5Dget_storage_size(dset_out); + PER((hssize_t)dsize_in,(hssize_t)dsize_out); + print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0); + } + else + print_dataset_info(dcpl_id,travt->objs[i].name,0); } - else - print_dataset_info(dcpl_id,travt->objs[i].name,0); - } - - if (apply_s==0 && options->verbose) - printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", - travt->objs[i].name, - (int)options->threshold); - - if (apply_f==0 && options->verbose) + + if (apply_s==0 && options->verbose) + printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", + travt->objs[i].name, + (int)options->threshold); + + if (apply_f==0 && options->verbose) printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); - - /*------------------------------------------------------------------------- - * copy attrs - *------------------------------------------------------------------------- - */ - if (copy_attr(dset_in,dset_out,options)<0) - goto error; - - /*close */ - if (H5Dclose(dset_out)<0) - goto error; - + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(dset_in,dset_out,options)<0) + goto error; + + /*close */ + if (H5Dclose(dset_out)<0) + goto error; + }/*H5T_STD_REF_OBJ*/ }/*can_read*/ + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Tclose(ftype_id)<0) + goto error; + if (H5Tclose(wtype_id)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Pclose(dcpl_out)<0) + goto error; + if (H5Sclose(f_space_id)<0) + goto error; + if (H5Dclose(dset_in)<0) + goto error; + } + /*------------------------------------------------------------------------- + * we do not have request for filter/chunking use H5Ocopy instead + *------------------------------------------------------------------------- + */ + else + { + hid_t pid; + + /* create property to pass copy options */ + if ( (pid = H5Pcreate(H5P_OBJECT_COPY)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * do the copy + *------------------------------------------------------------------------- + */ + + if (H5Ocopy(fidin, /* Source file or group identifier */ + travt->objs[i].name, /* Name of the source object to be copied */ + fidout, /* Destination file or group identifier */ + travt->objs[i].name, /* Name of the destination object */ + pid, /* Properties which apply to the copy */ + H5P_DEFAULT)<0) /* Properties which apply to the new hard link */ + goto error; + + /* close property */ + if (H5Pclose(pid)<0) + goto error; + + } /* end do we have request for filter/chunking */ -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - if (H5Tclose(ftype_id)<0) - goto error; - if (H5Tclose(wtype_id)<0) - goto error; - if (H5Pclose(dcpl_id)<0) - goto error; - if (H5Pclose(dcpl_out)<0) - goto error; - if (H5Sclose(f_space_id)<0) - goto error; - if (H5Dclose(dset_in)<0) - goto error; break; @@ -686,8 +755,6 @@ int do_copy_objects(hid_t fidin, /*------------------------------------------------------------------------- * the root is a special case, we get an ID for the root group * and copy its attributes using that ID - * it must be done last, because the attributes might contain references to - * objects in the object list *------------------------------------------------------------------------- */ @@ -827,8 +894,6 @@ int copy_attr(hid_t loc_in, */ if ( ! H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { - - /*------------------------------------------------------------------------- * read to memory *------------------------------------------------------------------------- @@ -894,6 +959,73 @@ error: return -1; } +/*------------------------------------------------------------------------- + * Function: do_hardlinks + * + * Purpose: duplicate hard links + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December, 10, 2003 + * + *------------------------------------------------------------------------- + */ + +static +int do_hardlinks(hid_t fidout,trav_table_t *travt) +{ + int i, j; + + for ( i = 0; i < travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + case H5G_GROUP: + + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++) + { + if (H5Glink(fidout, + H5L_TYPE_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name)<0) + return -1; + } + } + + break; + + case H5G_DATASET: + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++){ + if (H5Glink(fidout, + H5L_TYPE_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name)<0) + return -1; + } + } + break; + case H5G_TYPE: + case H5G_LINK: + case H5G_UDLINK: + + /*nothing to do */ + break; + + default: + + break; + } + } + + return 0; + +} diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 1d2ec98..5a0f8d6 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -106,10 +106,6 @@ void usage(void) printf("\n"); } - - - - /*------------------------------------------------------------------------- * Function: main * diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 7fce723..e01af2d 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -24,728 +24,8 @@ extern char *progname; static const char* MapIdToName(hid_t refobj_id, trav_table_t *travt); -static void close_obj(H5G_obj_t obj_type, hid_t obj_id); -static int copy_refs_attr(hid_t loc_in, - hid_t loc_out, - pack_opt_t *options, - trav_table_t *travt, - hid_t fidout /* for saving references */ - ); - -/*------------------------------------------------------------------------- - * Function: do_copy_refobjs - * - * Purpose: duplicate all referenced HDF5 objects in the file - * and create hard links - * - * Return: 0, ok, -1 no - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * - * Date: December, 10, 2003 - * - *------------------------------------------------------------------------- - */ - -int do_copy_refobjs(hid_t fidin, - hid_t fidout, - trav_table_t *travt, - pack_opt_t *options) /* repack options */ -{ - hid_t grp_in=-1; /* read group ID */ - hid_t grp_out=-1; /* write group ID */ - hid_t dset_in=-1; /* read dataset ID */ - hid_t dset_out=-1; /* write dataset ID */ - hid_t type_in=-1; /* named type ID */ - hid_t dcpl_id=-1; /* dataset creation property list ID */ - hid_t space_id=-1; /* space ID */ - hid_t ftype_id=-1; /* file type ID */ - hid_t wtype_id=-1; /* read/write type ID */ - size_t msize; /* size of type */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ - int next; /* external files */ - int i, j; - -/*------------------------------------------------------------------------- - * browse - *------------------------------------------------------------------------- - */ - - for ( i = 0; i < travt->nobjs; i++) - { - switch ( travt->objs[i].type ) - { - /*------------------------------------------------------------------------- - * H5G_GROUP - *------------------------------------------------------------------------- - */ - case H5G_GROUP: - - /*------------------------------------------------------------------------- - * check for hard links - *------------------------------------------------------------------------- - */ - - if (travt->objs[i].nlinks) - { - for ( j=0; j<travt->objs[i].nlinks; j++) - { - H5Glink(fidout, - H5L_TYPE_HARD, - travt->objs[i].name, - travt->objs[i].links[j].new_name); - } - } - - break; - - /*------------------------------------------------------------------------- - * H5G_DATASET - *------------------------------------------------------------------------- - */ - case H5G_DATASET: - - if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) - goto error; - if ((space_id=H5Dget_space(dset_in))<0) - goto error; - if ((ftype_id=H5Dget_type (dset_in))<0) - goto error; - if ((dcpl_id=H5Dget_create_plist(dset_in))<0) - goto error; - if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) - goto error; - if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) - goto error; - nelmts=1; - for (j=0; j<rank; j++) - nelmts*=dims[j]; - - if (options->use_native==1) - wtype_id = h5tools_get_native_type(ftype_id); - else - wtype_id = H5Tcopy(ftype_id); - - if ((msize=H5Tget_size(wtype_id))==0) - goto error; - -/*------------------------------------------------------------------------- - * check for external files - *------------------------------------------------------------------------- - */ - if ((next=H5Pget_external_count (dcpl_id))<0) - goto error; -/*------------------------------------------------------------------------- - * check if the dataset creation property list has filters that - * are not registered in the current configuration - * 1) the external filters GZIP and SZIP might not be available - * 2) the internal filters might be turned off - *------------------------------------------------------------------------- - */ - if (next==0 && h5tools_canreadf((NULL),dcpl_id)==1) - { -/*------------------------------------------------------------------------- - * test for a valid output dataset - *------------------------------------------------------------------------- - */ - dset_out = FAIL; - -/*------------------------------------------------------------------------- - * object references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - *------------------------------------------------------------------------- - */ - if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) - { - H5G_obj_t obj_type; - hid_t refobj_id; - hobj_ref_t *refbuf=NULL; /* buffer for object references */ - hobj_ref_t *buf=NULL; - const char* refname; - unsigned u; - - /*------------------------------------------------------------------------- - * read to memory - *------------------------------------------------------------------------- - */ - - if (nelmts) - { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - - if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) - goto error; - refbuf=HDmalloc((unsigned)nelmts*msize); - if ( refbuf==NULL){ - error_msg(progname, "cannot allocate memory\n" ); - goto error; - } - for ( u=0; u<nelmts; u++) - { - H5E_BEGIN_TRY { - if ((refobj_id = H5Rdereference(dset_in,H5R_OBJECT,&buf[u]))<0) - continue; - } H5E_END_TRY; - /* get the name. a valid name could only occur in the - second traversal of the file */ - if ((refname=MapIdToName(refobj_id,travt))!=NULL) - { - /* create the reference, -1 parameter for objects */ - if (H5Rcreate(&refbuf[u],fidout,refname,H5R_OBJECT,-1)<0) - goto error; - if (options->verbose) - printf("object <%s> object reference created to <%s>\n", - travt->objs[i].name, - refname); - }/*refname*/ - close_obj(obj_type,refobj_id); - }/* u */ - }/*nelmts*/ - - /*------------------------------------------------------------------------- - * create/write dataset/close - *------------------------------------------------------------------------- - */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) - goto error; - if (nelmts) { - if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) - goto error; - } - - if (buf) - free(buf); - if (refbuf) - free(refbuf); - - }/*H5T_STD_REF_OBJ*/ - -/*------------------------------------------------------------------------- - * dataset region references - *------------------------------------------------------------------------- - */ - else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) - { - H5G_obj_t obj_type; - hid_t refobj_id; - hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ - hdset_reg_ref_t *buf=NULL; /* output buffer */ - const char* refname; - unsigned u; - - /*------------------------------------------------------------------------- - * read input to memory - *------------------------------------------------------------------------- - */ - if (nelmts) - { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) - goto error; - - /*------------------------------------------------------------------------- - * create output - *------------------------------------------------------------------------- - */ - - refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ - if ( refbuf==NULL){ - error_msg(progname, "cannot allocate memory\n" ); - goto error; - } - for ( u=0; u<nelmts; u++) - { - H5E_BEGIN_TRY { - if ((refobj_id = H5Rdereference(dset_in,H5R_DATASET_REGION,&buf[u]))<0) - continue; - } H5E_END_TRY; - - /* get the name. a valid name could only occur in the - second traversal of the file */ - if ((refname=MapIdToName(refobj_id,travt))!=NULL) - { - hid_t region_id; /* region id of the referenced dataset */ - if ((region_id = H5Rget_region(dset_in,H5R_DATASET_REGION,&buf[u]))<0) - goto error; - /* create the reference, we need the space_id */ - if (H5Rcreate(&refbuf[u],fidout,refname,H5R_DATASET_REGION,region_id)<0) - goto error; - if (H5Sclose(region_id)<0) - goto error; - if (options->verbose) - printf("object <%s> region reference created to <%s>\n", - travt->objs[i].name, - refname); - }/*refname*/ - close_obj(obj_type,refobj_id); - }/* u */ - }/*nelmts*/ - - /*------------------------------------------------------------------------- - * create/write dataset/close - *------------------------------------------------------------------------- - */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) - goto error; - if (nelmts) { - if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) - goto error; - } - - if (buf) - free(buf); - if (refbuf) - free(refbuf); - } /* H5T_STD_REF_DSETREG */ - - -/*------------------------------------------------------------------------- - * not references, open previously created object in 1st traversal - *------------------------------------------------------------------------- - */ - else - { - if ((dset_out=H5Dopen(fidout,travt->objs[i].name))<0) - goto error; - } - - assert(dset_out!=FAIL); - -/*------------------------------------------------------------------------- - * copy referenced objects in attributes - *------------------------------------------------------------------------- - */ - if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0) - goto error; - - -/*------------------------------------------------------------------------- - * check for hard links - *------------------------------------------------------------------------- - */ - if (travt->objs[i].nlinks) - { - for ( j=0; j<travt->objs[i].nlinks; j++){ - H5Glink(fidout, - H5L_TYPE_HARD, - travt->objs[i].name, - travt->objs[i].links[j].new_name); - } - } - - if (H5Dclose(dset_out)<0) - goto error; - - }/*can_read*/ - - /*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - - if (H5Tclose(ftype_id)<0) - goto error; - if (H5Tclose(wtype_id)<0) - goto error; - if (H5Pclose(dcpl_id)<0) - goto error; - if (H5Sclose(space_id)<0) - goto error; - if (H5Dclose(dset_in)<0) - goto error; - - break; - - /*------------------------------------------------------------------------- - * H5G_TYPE - *------------------------------------------------------------------------- - */ - case H5G_TYPE: - - if ((type_in = H5Topen (fidin,travt->objs[i].name))<0) - goto error; - - if (H5Tclose(type_in)<0) - goto error; - - break; - - /*------------------------------------------------------------------------- - * H5G_LINK - * H5G_UDLINK - *------------------------------------------------------------------------- - */ - - case H5G_LINK: - case H5G_UDLINK: - - /*nothing to do */ - break; - - default: - - break; - } - } - - - -/*------------------------------------------------------------------------- - * the root is a special case, we get an ID for the root group - * and copy its attributes using that ID - * it must be done last, because the attributes might contain references to - * objects in the object list - *------------------------------------------------------------------------- - */ - - if ((grp_out = H5Gopen(fidout,"/"))<0) - goto error; - - if ((grp_in = H5Gopen(fidin,"/"))<0) - goto error; - - if (copy_refs_attr(grp_in,grp_out,options,travt,fidout)<0) - goto error; - - if (H5Gclose(grp_out)<0) - goto error; - if (H5Gclose(grp_in)<0) - goto error; - - return 0; - -error: - H5E_BEGIN_TRY { - H5Gclose(grp_in); - H5Gclose(grp_out); - H5Pclose(dcpl_id); - H5Sclose(space_id); - H5Dclose(dset_in); - H5Dclose(dset_out); - H5Tclose(ftype_id); - H5Tclose(wtype_id); - H5Tclose(type_in); - } H5E_END_TRY; - return -1; - -} - - -/*------------------------------------------------------------------------- - * Function: copy_refs_attr - * - * Purpose: duplicate all referenced HDF5 located in attributes - * relative to LOC_IN, which is obtained either from - * loc_id = H5Gopen( fid, name); - * loc_id = H5Dopen( fid, name); - * loc_id = H5Topen( fid, name); - * - * Return: 0, ok, -1 no - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * - * Date: October, 28, 2003 - * - *------------------------------------------------------------------------- - */ - -static int copy_refs_attr(hid_t loc_in, - hid_t loc_out, - pack_opt_t *options, - trav_table_t *travt, - hid_t fidout /* for saving references */ - ) -{ - hid_t attr_id; /* attr ID */ - hid_t attr_out; /* attr ID */ - hid_t space_id; /* space ID */ - hid_t ftype_id; /* file type ID */ - hid_t wtype_id; /* read/write type ID */ - size_t msize; /* size of type */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ - char name[255]; - int n, j; - unsigned u; - - if ((n = H5Aget_num_attrs(loc_in))<0) - goto error; - - for ( u = 0; u < (unsigned)n; u++) - { - -/*------------------------------------------------------------------------- - * open - *------------------------------------------------------------------------- - */ - /* open attribute */ - if ((attr_id = H5Aopen_idx(loc_in, u))<0) - goto error; - - /* get name */ - if (H5Aget_name( attr_id, 255, name )<0) - goto error; - - /* get the file datatype */ - if ((ftype_id = H5Aget_type( attr_id )) < 0 ) - goto error; - - /* get the dataspace handle */ - if ((space_id = H5Aget_space( attr_id )) < 0 ) - goto error; - - /* get dimensions */ - if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) - goto error; - - - /*------------------------------------------------------------------------- - * elements - *------------------------------------------------------------------------- - */ - nelmts=1; - for (j=0; j<rank; j++) - nelmts*=dims[j]; - - if (options->use_native==1) - wtype_id = h5tools_get_native_type(ftype_id); - else - wtype_id = H5Tcopy(ftype_id); - - if ((msize=H5Tget_size(wtype_id))==0) - goto error; - -/*------------------------------------------------------------------------- - * object references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - *------------------------------------------------------------------------- - */ - if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) - { - H5G_obj_t obj_type; - hid_t refobj_id; - hobj_ref_t *refbuf=NULL; - unsigned k; - const char* refname; - hobj_ref_t *buf=NULL; - - /*------------------------------------------------------------------------- - * read input to memory - *------------------------------------------------------------------------- - */ - - if (nelmts) - { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Aread(attr_id,wtype_id,buf)<0) - goto error; - - if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0) - goto error; - refbuf=HDmalloc((unsigned)nelmts*msize); - if ( refbuf==NULL){ - error_msg(progname, "cannot allocate memory\n" ); - goto error; - } - for ( k=0; k<nelmts; k++) - { - H5E_BEGIN_TRY { - if ((refobj_id = H5Rdereference(attr_id,H5R_OBJECT,&buf[k]))<0) - goto error; - } H5E_END_TRY; - /* get the name. a valid name could only occur in the - second traversal of the file */ - if ((refname=MapIdToName(refobj_id,travt))!=NULL) - { - /* create the reference */ - if (H5Rcreate(&refbuf[k],fidout,refname,H5R_OBJECT,-1)<0) - goto error; - if (options->verbose) - printf("object <%s> reference created to <%s>\n",name,refname); - } - close_obj(obj_type,refobj_id); - }/* k */ - }/*nelmts*/ - - /*------------------------------------------------------------------------- - * copy - *------------------------------------------------------------------------- - */ - - if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) - goto error; - if (nelmts) { - if(H5Awrite(attr_out,wtype_id,refbuf)<0) - goto error; - } - - if (H5Aclose(attr_out)<0) - goto error; - - if (refbuf) - free(refbuf); - if (buf) - free(buf); - - }/*H5T_STD_REF_OBJ*/ - -/*------------------------------------------------------------------------- - * dataset region references - *------------------------------------------------------------------------- - */ - else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) - { - H5G_obj_t obj_type; - hid_t refobj_id; - hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ - hdset_reg_ref_t *buf=NULL; /* output buffer */ - const char* refname; - unsigned k; - - /*------------------------------------------------------------------------- - * read input to memory - *------------------------------------------------------------------------- - */ - - if (nelmts) - { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Aread(attr_id,wtype_id,buf)<0) - goto error; - if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0) - goto error; - - /*------------------------------------------------------------------------- - * create output - *------------------------------------------------------------------------- - */ - - refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ - if ( refbuf==NULL){ - error_msg(progname, "cannot allocate memory\n" ); - goto error; - } - for ( k=0; k<nelmts; k++) - { - H5E_BEGIN_TRY { - if ((refobj_id = H5Rdereference(attr_id,H5R_DATASET_REGION,&buf[k]))<0) - continue; - } H5E_END_TRY; - /* get the name. a valid name could only occur in the - second traversal of the file */ - if ((refname=MapIdToName(refobj_id,travt))!=NULL) - { - hid_t region_id; /* region id of the referenced dataset */ - if ((region_id = H5Rget_region(attr_id,H5R_DATASET_REGION,&buf[k]))<0) - goto error; - /* create the reference, we need the space_id */ - if (H5Rcreate(&refbuf[k],fidout,refname,H5R_DATASET_REGION,region_id)<0) - goto error; - if (H5Sclose(region_id)<0) - goto error; - if (options->verbose) - printf("object <%s> region reference created to <%s>\n",name,refname); - } - close_obj(obj_type,refobj_id); - }/* k */ - }/*nelmts */ - - /*------------------------------------------------------------------------- - * copy - *------------------------------------------------------------------------- - */ - - if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) - goto error; - if (nelmts) { - if(H5Awrite(attr_out,wtype_id,refbuf)<0) - goto error; - } - if (H5Aclose(attr_out)<0) - goto error; - if (refbuf) - free(refbuf); - if (buf) - free(buf); - } /* H5T_STD_REF_DSETREG */ - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - - if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(wtype_id)<0) goto error; - if (H5Sclose(space_id)<0) goto error; - if (H5Aclose(attr_id)<0) goto error; - } /* u */ - - return 0; - -error: - H5E_BEGIN_TRY { - H5Tclose(ftype_id); - H5Tclose(wtype_id); - H5Sclose(space_id); - H5Aclose(attr_id); - H5Aclose(attr_out); - } H5E_END_TRY; - return -1; -} - -/*------------------------------------------------------------------------- - * Function: close_obj - * - * Purpose: Auxiliary function to close an object - * - *------------------------------------------------------------------------- - */ - -static void close_obj(H5G_obj_t obj_type, hid_t obj_id) -{ - H5E_BEGIN_TRY - { - switch (obj_type) - { - case H5G_GROUP: - H5Gclose(obj_id); - break; - case H5G_DATASET: - H5Dclose(obj_id); - break; - case H5G_TYPE: - H5Tclose(obj_id); - break; - default: - break; - } - } H5E_END_TRY; -} /*------------------------------------------------------------------------- * Function: MapIdToName |