diff options
Diffstat (limited to 'tools/h5repack/h5repack_copy.c')
| -rw-r--r-- | tools/h5repack/h5repack_copy.c | 773 |
1 files changed, 765 insertions, 8 deletions
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 6b3e4d0..5dbb5ce 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -23,18 +23,23 @@ extern char *progname; +static int do_create_refs; + #define PER(A,B) { per = 0; \ if (A!=0) \ - per = (float)fabs(1-( (float)B / (float)A )); \ + per = (float)fabs( ((float)B - (float)A) / (float) A ); \ } #define FORMAT_OBJ " %-21s %s\n" /* obj type, name */ #define FORMAT_OBJ_ATTR " %-21s %s\n" /* obj type, name */ - -static -int do_hardlinks(hid_t fidout,trav_table_t *travt); +/* local functions */ +static int do_hardlinks(hid_t fidout,trav_table_t *travt); +static void close_obj(H5G_obj_t obj_type, hid_t obj_id); +static int do_copy_refobjs(hid_t fidin, hid_t fidout,trav_table_t *travt,pack_opt_t *options); +static int copy_refs_attr(hid_t loc_in,hid_t loc_out,pack_opt_t *options,trav_table_t *travt,hid_t fidout); +static const char* MapIdToName(hid_t refobj_id,trav_table_t *travt); /*------------------------------------------------------------------------- * Function: print_dataset_info @@ -88,7 +93,6 @@ static void print_dataset_info(hid_t dcpl_id, NULL); #endif /* H5_WANT_H5_V1_6_COMPAT */ - switch (filtn) { default: @@ -207,6 +211,18 @@ int copy_objects(const char* fnamein, } /*------------------------------------------------------------------------- + * do the copy of referenced objects in a second sweep if needed (references + * detected in the first traversal) + *------------------------------------------------------------------------- + */ + if (do_create_refs){ + if(do_copy_refobjs(fidin,fidout,travt,options)<0) { + error_msg(progname, "<%s>: Could not copy data to: %s\n", fnamein, fnameout); + goto out; + } + } + +/*------------------------------------------------------------------------- * create hard links *------------------------------------------------------------------------- */ @@ -265,6 +281,12 @@ out: * October 2006: Read/write using the file type by default. * Read/write by hyperslabs for big datasets. * + * November 2006: Use H5Ocopy in the copy of objects. The logic for using + * H5Ocopy or not is if the input DCPL has filters or non default layout or these are + * requested by the user then use read/write else use H5Ocopy. + * A detection is made for the cases where the recreation of references is needed + * in a second sweep of the file + * *------------------------------------------------------------------------- */ @@ -444,7 +466,11 @@ int do_copy_objects(hid_t fidin, * in a second traversal of the output file *------------------------------------------------------------------------- */ - if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) + if (H5T_REFERENCE==H5Tget_class(wtype_id)) + { + do_create_refs = 1; + } + else /* H5T_REFERENCE */ { /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); @@ -629,8 +655,8 @@ int do_copy_objects(hid_t fidin, if (H5Dclose(dset_out)<0) goto error; - }/*H5T_STD_REF_OBJ*/ - }/*can_read*/ + }/*H5T_REFERENCE*/ + }/*h5tools_canreadf*/ /*------------------------------------------------------------------------- @@ -1029,3 +1055,734 @@ int do_hardlinks(hid_t fidout,trav_table_t *travt) return 0; } + + + +/*------------------------------------------------------------------------- + * Function: do_copy_refobjs + * + * Purpose: duplicate all referenced HDF5 objects in the file in a second traversal + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December, 10, 2003 + * + *------------------------------------------------------------------------- + */ + +static +int do_copy_refobjs(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options) /* repack options */ +{ + hid_t grp_in=-1; /* read group ID */ + hid_t grp_out=-1; /* write group ID */ + hid_t dset_in=-1; /* read dataset ID */ + hid_t dset_out=-1; /* write dataset ID */ + hid_t type_in=-1; /* named type ID */ + hid_t dcpl_id=-1; /* dataset creation property list ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file type ID */ + hid_t wtype_id=-1; /* read/write type ID */ + size_t msize; /* size of type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + int next; /* external files */ + int i, j; + +/*------------------------------------------------------------------------- + * browse + *------------------------------------------------------------------------- + */ + + for ( i = 0; i < travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + + case H5G_GROUP: + + break; + + /*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + case H5G_DATASET: + + if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) + goto error; + if ((space_id=H5Dget_space(dset_in))<0) + goto error; + if ((ftype_id=H5Dget_type (dset_in))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) + goto error; + if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) + goto error; + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); + + if ((msize=H5Tget_size(wtype_id))==0) + goto error; + +/*------------------------------------------------------------------------- + * check for external files + *------------------------------------------------------------------------- + */ + if ((next=H5Pget_external_count (dcpl_id))<0) + goto error; +/*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- + */ + if (next==0 && h5tools_canreadf((NULL),dcpl_id)==1) + { +/*------------------------------------------------------------------------- + * test for a valid output dataset + *------------------------------------------------------------------------- + */ + dset_out = FAIL; + +/*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + *------------------------------------------------------------------------- + */ + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hobj_ref_t *refbuf=NULL; /* buffer for object references */ + hobj_ref_t *buf=NULL; + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + error_msg(progname, "cannot read into memory\n" ); + goto error; + } + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + + if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) + goto error; + refbuf=HDmalloc((unsigned)nelmts*msize); + if ( refbuf==NULL){ + error_msg(progname, "cannot allocate memory\n" ); + goto error; + } + for ( u=0; u<nelmts; u++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(dset_in,H5R_OBJECT,&buf[u]))<0) + continue; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + /* create the reference, -1 parameter for objects */ + if (H5Rcreate(&refbuf[u],fidout,refname,H5R_OBJECT,-1)<0) + goto error; + if (options->verbose) + printf("object <%s> object reference created to <%s>\n", + travt->objs[i].name, + refname); + }/*refname*/ + close_obj(obj_type,refobj_id); + }/* u */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) + goto error; + if (nelmts) { + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + goto error; + } + + if (buf) + free(buf); + if (refbuf) + free(refbuf); + + }/*H5T_STD_REF_OBJ*/ + +/*------------------------------------------------------------------------- + * dataset region references + *------------------------------------------------------------------------- + */ + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ + hdset_reg_ref_t *buf=NULL; /* output buffer */ + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + error_msg(progname, "cannot read into memory\n" ); + goto error; + } + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) + goto error; + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + + refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ + if ( refbuf==NULL){ + error_msg(progname, "cannot allocate memory\n" ); + goto error; + } + for ( u=0; u<nelmts; u++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(dset_in,H5R_DATASET_REGION,&buf[u]))<0) + continue; + } H5E_END_TRY; + + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + hid_t region_id; /* region id of the referenced dataset */ + if ((region_id = H5Rget_region(dset_in,H5R_DATASET_REGION,&buf[u]))<0) + goto error; + /* create the reference, we need the space_id */ + if (H5Rcreate(&refbuf[u],fidout,refname,H5R_DATASET_REGION,region_id)<0) + goto error; + if (H5Sclose(region_id)<0) + goto error; + if (options->verbose) + printf("object <%s> region reference created to <%s>\n", + travt->objs[i].name, + refname); + }/*refname*/ + close_obj(obj_type,refobj_id); + }/* u */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) + goto error; + if (nelmts) { + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + goto error; + } + + if (buf) + free(buf); + if (refbuf) + free(refbuf); + } /* H5T_STD_REF_DSETREG */ + + +/*------------------------------------------------------------------------- + * not references, open previously created object in 1st traversal + *------------------------------------------------------------------------- + */ + else + { + if ((dset_out=H5Dopen(fidout,travt->objs[i].name))<0) + goto error; + } + + assert(dset_out!=FAIL); + +/*------------------------------------------------------------------------- + * copy referenced objects in attributes + *------------------------------------------------------------------------- + */ + if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0) + goto error; + + + + if (H5Dclose(dset_out)<0) + goto error; + + }/*can_read*/ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id)<0) + goto error; + if (H5Tclose(wtype_id)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_in)<0) + goto error; + + + break; + + + case H5G_TYPE: + case H5G_LINK: + case H5G_UDLINK: + + /*nothing to do */ + break; + + default: + + break; + } + } + + +/*------------------------------------------------------------------------- + * the root is a special case, we get an ID for the root group + * and copy its attributes using that ID + * it must be done last, because the attributes might contain references to + * objects in the object list + *------------------------------------------------------------------------- + */ + + if ((grp_out = H5Gopen(fidout,"/"))<0) + goto error; + + if ((grp_in = H5Gopen(fidin,"/"))<0) + goto error; + + if (copy_refs_attr(grp_in,grp_out,options,travt,fidout)<0) + goto error; + + if (H5Gclose(grp_out)<0) + goto error; + if (H5Gclose(grp_in)<0) + goto error; + + + return 0; + +error: + H5E_BEGIN_TRY { + H5Gclose(grp_in); + H5Gclose(grp_out); + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_in); + H5Dclose(dset_out); + H5Tclose(ftype_id); + H5Tclose(wtype_id); + H5Tclose(type_in); + } H5E_END_TRY; + return -1; + +} + + + +/*------------------------------------------------------------------------- + * Function: copy_refs_attr + * + * Purpose: duplicate all referenced HDF5 objects located in attributes + * relative to LOC_IN, which is obtained either from + * loc_id = H5Gopen( fid, name); + * loc_id = H5Dopen( fid, name); + * loc_id = H5Topen( fid, name); + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 28, 2003 + * + *------------------------------------------------------------------------- + */ + +static int copy_refs_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options, + trav_table_t *travt, + hid_t fidout /* for saving references */ + ) +{ + hid_t attr_id; /* attr ID */ + hid_t attr_out; /* attr ID */ + hid_t space_id; /* space ID */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + int n, j; + unsigned u; + + if ((n = H5Aget_num_attrs(loc_in))<0) + goto error; + + for ( u = 0; u < (unsigned)n; u++) + { + +/*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + /* open attribute */ + if ((attr_id = H5Aopen_idx(loc_in, u))<0) + goto error; + + /* get name */ + if (H5Aget_name( attr_id, 255, name )<0) + goto error; + + /* get the file datatype */ + if ((ftype_id = H5Aget_type( attr_id )) < 0 ) + goto error; + + /* get the dataspace handle */ + if ((space_id = H5Aget_space( attr_id )) < 0 ) + goto error; + + /* get dimensions */ + if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) + goto error; + + + /*------------------------------------------------------------------------- + * elements + *------------------------------------------------------------------------- + */ + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); + + if ((msize=H5Tget_size(wtype_id))==0) + goto error; + +/*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + *------------------------------------------------------------------------- + */ + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hobj_ref_t *refbuf=NULL; + unsigned k; + const char* refname; + hobj_ref_t *buf=NULL; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + error_msg(progname, "cannot read into memory\n" ); + goto error; + } + if (H5Aread(attr_id,wtype_id,buf)<0) + goto error; + + if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0) + goto error; + refbuf=HDmalloc((unsigned)nelmts*msize); + if ( refbuf==NULL){ + error_msg(progname, "cannot allocate memory\n" ); + goto error; + } + for ( k=0; k<nelmts; k++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(attr_id,H5R_OBJECT,&buf[k]))<0) + goto error; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + /* create the reference */ + if (H5Rcreate(&refbuf[k],fidout,refname,H5R_OBJECT,-1)<0) + goto error; + if (options->verbose) + printf("object <%s> reference created to <%s>\n",name,refname); + } + close_obj(obj_type,refobj_id); + }/* k */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) + goto error; + if (nelmts) { + if(H5Awrite(attr_out,wtype_id,refbuf)<0) + goto error; + } + + if (H5Aclose(attr_out)<0) + goto error; + + if (refbuf) + free(refbuf); + if (buf) + free(buf); + + }/*H5T_STD_REF_OBJ*/ + +/*------------------------------------------------------------------------- + * dataset region references + *------------------------------------------------------------------------- + */ + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ + hdset_reg_ref_t *buf=NULL; /* output buffer */ + const char* refname; + unsigned k; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + error_msg(progname, "cannot read into memory\n" ); + goto error; + } + if (H5Aread(attr_id,wtype_id,buf)<0) + goto error; + if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0) + goto error; + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + + refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ + if ( refbuf==NULL){ + error_msg(progname, "cannot allocate memory\n" ); + goto error; + } + for ( k=0; k<nelmts; k++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(attr_id,H5R_DATASET_REGION,&buf[k]))<0) + continue; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + hid_t region_id; /* region id of the referenced dataset */ + if ((region_id = H5Rget_region(attr_id,H5R_DATASET_REGION,&buf[k]))<0) + goto error; + /* create the reference, we need the space_id */ + if (H5Rcreate(&refbuf[k],fidout,refname,H5R_DATASET_REGION,region_id)<0) + goto error; + if (H5Sclose(region_id)<0) + goto error; + if (options->verbose) + printf("object <%s> region reference created to <%s>\n",name,refname); + } + close_obj(obj_type,refobj_id); + }/* k */ + }/*nelmts */ + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) + goto error; + if (nelmts) { + if(H5Awrite(attr_out,wtype_id,refbuf)<0) + goto error; + } + if (H5Aclose(attr_out)<0) + goto error; + if (refbuf) + free(refbuf); + if (buf) + free(buf); + } /* H5T_STD_REF_DSETREG */ + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id)<0) goto error; + if (H5Tclose(wtype_id)<0) goto error; + if (H5Sclose(space_id)<0) goto error; + if (H5Aclose(attr_id)<0) goto error; + } /* u */ + + return 0; + +error: + H5E_BEGIN_TRY { + H5Tclose(ftype_id); + H5Tclose(wtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + } H5E_END_TRY; + return -1; +} + +/*------------------------------------------------------------------------- + * Function: close_obj + * + * Purpose: Auxiliary function to close an object + * + *------------------------------------------------------------------------- + */ + +static void close_obj(H5G_obj_t obj_type, hid_t obj_id) +{ + H5E_BEGIN_TRY + { + switch (obj_type) + { + case H5G_GROUP: + H5Gclose(obj_id); + break; + case H5G_DATASET: + H5Dclose(obj_id); + break; + case H5G_TYPE: + H5Tclose(obj_id); + break; + default: + break; + } + } H5E_END_TRY; +} + +/*------------------------------------------------------------------------- + * Function: MapIdToName + * + * Purpose: map an object ID to a name + * + *------------------------------------------------------------------------- + */ + +static const char* MapIdToName(hid_t refobj_id, + trav_table_t *travt) +{ + hid_t id; + hid_t fid; + H5G_stat_t refstat; /* Stat for the refobj id */ + H5G_stat_t objstat; /* Stat for objects in the file */ + int i; + + /* obtain information to identify the referenced object uniquely */ + if(H5Gget_objinfo(refobj_id, ".", 0, &refstat) <0) + return NULL; + + /* obtains the file ID given an object ID. This ID must be closed */ + if ((fid = H5Iget_file_id(refobj_id))<0) + { + return NULL; + } + + /* linear search */ + for ( i=0; i<travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + default: + break; + + /*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + + case H5G_DATASET: + + if ((id = H5Dopen(fid,travt->objs[i].name))<0) + return NULL; + if(H5Gget_objinfo(id, ".", 0, &objstat) <0) + return NULL; + if (H5Dclose(id)<0) + return NULL; + if (!HDmemcmp(&refstat.fileno, &objstat.fileno, sizeof(refstat.fileno)) && !HDmemcmp(&refstat.objno, &objstat.objno, sizeof(refstat.objno))) + { + H5Fclose(fid); + return travt->objs[i].name; + } + break; + } /* switch */ + } /* i */ + + if (H5Fclose(fid)<0) + return NULL; + + return NULL; +} + |