summaryrefslogtreecommitdiffstats
path: root/tools/h5repack/h5repack_copy.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/h5repack/h5repack_copy.c')
-rw-r--r--tools/h5repack/h5repack_copy.c604
1 files changed, 368 insertions, 236 deletions
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 03edd01..6b3e4d0 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -33,6 +33,8 @@ extern char *progname;
#define FORMAT_OBJ_ATTR " %-21s %s\n" /* obj type, name */
+static
+int do_hardlinks(hid_t fidout,trav_table_t *travt);
/*-------------------------------------------------------------------------
* Function: print_dataset_info
@@ -148,12 +150,8 @@ static void print_dataset_info(hid_t dcpl_id,
strcat(str,temp);
printf(FORMAT_OBJ,str,objname);
}
-
-
-
}
-
/*-------------------------------------------------------------------------
* Function: copy_objects
*
@@ -209,11 +207,11 @@ int copy_objects(const char* fnamein,
}
/*-------------------------------------------------------------------------
- * do the copy of referenced objects
- * and create hard links
+ * create hard links
*-------------------------------------------------------------------------
*/
- if(do_copy_refobjs(fidin,fidout,travt,options)<0) {
+
+ if(do_hardlinks(fidout,travt)<0) {
error_msg(progname, "<%s>: Could not copy data to: %s\n", fnamein, fnameout);
goto out;
}
@@ -299,6 +297,8 @@ int do_copy_objects(hid_t fidin,
void *buf=NULL; /* buffer for raw data */
void *sm_buf=NULL; /* buffer for raw data */
int i, j;
+ int nfilters; /* number of filters in DCPL */
+ H5D_layout_t layout;
/*-------------------------------------------------------------------------
* copy the suppplied object list
@@ -351,268 +351,337 @@ int do_copy_objects(hid_t fidin,
*-------------------------------------------------------------------------
*/
case H5G_DATASET:
-
+
+/*-------------------------------------------------------------------------
+ * early check for filters or a non default layout
+ *-------------------------------------------------------------------------
+ */
if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0)
goto error;
- if ((f_space_id=H5Dget_space(dset_in))<0)
- goto error;
- if ((ftype_id=H5Dget_type (dset_in))<0)
- goto error;
if ((dcpl_id=H5Dget_create_plist(dset_in))<0)
+ goto error;
+
+ /* get information about input filters */
+ if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
goto error;
- if ((dcpl_out = H5Pcopy (dcpl_id))<0)
- goto error;
- if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0)
- goto error;
- HDmemset(dims, 0, sizeof dims);
- if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0)
- goto error;
- nelmts=1;
- for (j=0; j<rank; j++)
- nelmts*=dims[j];
-
- if (options->use_native==1)
- wtype_id = h5tools_get_native_type(ftype_id);
- else
- wtype_id = H5Tcopy(ftype_id);
-
- if ((msize=H5Tget_size(wtype_id))==0)
- goto error;
-
-/*-------------------------------------------------------------------------
- * check for external files
- *-------------------------------------------------------------------------
- */
- if ((next=H5Pget_external_count (dcpl_id))<0)
+ if ((layout = H5Pget_layout(dcpl_id))<0)
goto error;
- if (next)
- fprintf(stderr," <warning: %s has external files, ignoring read...>\n",
- travt->objs[i].name );
-
-/*-------------------------------------------------------------------------
- * check if the dataset creation property list has filters that
- * are not registered in the current configuration
- * 1) the external filters GZIP and SZIP might not be available
- * 2) the internal filters might be turned off
- *-------------------------------------------------------------------------
- */
- if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1)
- {
- apply_s=1;
- apply_f=1;
-
+ if (H5Dclose(dset_in)<0)
+ goto error;
+ if (H5Pclose(dcpl_id)<0)
+ goto error;
+
/*-------------------------------------------------------------------------
- * references are a special case
- * we cannot just copy the buffers, but instead we recreate the reference
- * in a second traversal of the output file
+ * check if we should use H5Ocopy or not
+ * if the DCPL has filters or a non default layout, we read/write the object
+ * otherwise we do a copy using H5Ocopy
*-------------------------------------------------------------------------
*/
- if ( (H5T_REFERENCE!=H5Tget_class(wtype_id)))
+
+ if (nfilters || layout!=H5D_CONTIGUOUS
+ /* does the dcpl has filters or non default layout ? */
+ ||
+ options->op_tbl->nelems
+ /* do we have input request for filter/chunking */
+ ||
+ options->all_filter==1 || options->all_layout==1
+ )
{
- /* get the storage size of the input dataset */
- dsize_in=H5Dget_storage_size(dset_in);
-
- /* check for datasets too small */
- if (nelmts*msize < options->threshold )
- apply_s=0;
+ if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0)
+ goto error;
+ if ((f_space_id=H5Dget_space(dset_in))<0)
+ goto error;
+ if ((ftype_id=H5Dget_type (dset_in))<0)
+ goto error;
+ if ((dcpl_id=H5Dget_create_plist(dset_in))<0)
+ goto error;
+ if ((dcpl_out = H5Pcopy (dcpl_id))<0)
+ goto error;
+ if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0)
+ goto error;
+ HDmemset(dims, 0, sizeof dims);
+ if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0)
+ goto error;
+ nelmts=1;
+ for (j=0; j<rank; j++)
+ nelmts*=dims[j];
- /* apply the filter */
- if (apply_s){
- if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0)
- goto error;
- }
+ if (options->use_native==1)
+ wtype_id = h5tools_get_native_type(ftype_id);
+ else
+ wtype_id = H5Tcopy(ftype_id);
- /*-------------------------------------------------------------------------
- * create the output dataset;
- * disable error checking in case the dataset cannot be created with the
- * modified dcpl; in that case use the original instead
- *-------------------------------------------------------------------------
+ if ((msize=H5Tget_size(wtype_id))==0)
+ goto error;
+
+ /*-------------------------------------------------------------------------
+ * check for external files
+ *-------------------------------------------------------------------------
+ */
+ if ((next=H5Pget_external_count (dcpl_id))<0)
+ goto error;
+
+ if (next)
+ fprintf(stderr," <warning: %s has external files, ignoring read...>\n",
+ travt->objs[i].name );
+
+ /*-------------------------------------------------------------------------
+ * check if the dataset creation property list has filters that
+ * are not registered in the current configuration
+ * 1) the external filters GZIP and SZIP might not be available
+ * 2) the internal filters might be turned off
+ *-------------------------------------------------------------------------
*/
- H5E_BEGIN_TRY {
- dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out);
- } H5E_END_TRY;
- if (dset_out==FAIL)
+ if (next==0 && h5tools_canreadf((travt->objs[i].name),dcpl_id)==1)
{
- if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0)
- goto error;
- apply_f=0;
- }
-
- /*-------------------------------------------------------------------------
- * read/write
- *-------------------------------------------------------------------------
- */
- if (nelmts)
- {
- size_t need = (size_t)(nelmts*msize); /* bytes needed */
- if ( need < H5TOOLS_MALLOCSIZE )
- buf = HDmalloc(need);
-
- if (buf != NULL )
- {
- if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
- goto error;
- if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
- goto error;
- }
+ apply_s=1;
+ apply_f=1;
- else /* possibly not enough memory, read/write by hyperslabs */
-
+ /*-------------------------------------------------------------------------
+ * references are a special case
+ * we cannot just copy the buffers, but instead we recreate the reference
+ * in a second traversal of the output file
+ *-------------------------------------------------------------------------
+ */
+ if ( (H5T_REFERENCE!=H5Tget_class(wtype_id)))
{
- size_t p_type_nbytes = msize; /*size of memory type */
- hsize_t p_nelmts = nelmts; /*total selected elmts */
- hsize_t elmtno; /*counter */
- int carry; /*counter carry value */
- unsigned int vl_data = 0; /*contains VL datatypes */
+ /* get the storage size of the input dataset */
+ dsize_in=H5Dget_storage_size(dset_in);
- /* stripmine info */
- hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
- hsize_t sm_nbytes; /*bytes per stripmine */
- hsize_t sm_nelmts; /*elements per stripmine*/
- hid_t sm_space; /*stripmine data space */
+ /* check for datasets too small */
+ if (nelmts*msize < options->threshold )
+ apply_s=0;
- /* hyperslab info */
- hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */
- hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */
- hsize_t hs_nelmts; /*elements in request */
- hsize_t zero[8]; /*vector of zeros */
+ /* apply the filter */
+ if (apply_s){
+ if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0)
+ goto error;
+ }
- /* check if we have VL data in the dataset's datatype */
- if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE)
- vl_data = TRUE;
-
- /*
- * determine the strip mine size and allocate a buffer. The strip mine is
- * a hyperslab whose size is manageable.
+ /*-------------------------------------------------------------------------
+ * create the output dataset;
+ * disable error checking in case the dataset cannot be created with the
+ * modified dcpl; in that case use the original instead
+ *-------------------------------------------------------------------------
*/
- sm_nbytes = p_type_nbytes;
-
- for (i = rank; i > 0; --i) {
- sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
- sm_nbytes *= sm_size[i - 1];
- assert(sm_nbytes > 0);
+ H5E_BEGIN_TRY {
+ dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out);
+ } H5E_END_TRY;
+ if (dset_out==FAIL)
+ {
+ if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0)
+ goto error;
+ apply_f=0;
}
- sm_buf = HDmalloc((size_t)sm_nbytes);
-
- sm_nelmts = sm_nbytes / p_type_nbytes;
- sm_space = H5Screate_simple(1, &sm_nelmts, NULL);
- /* the stripmine loop */
- memset(hs_offset, 0, sizeof hs_offset);
- memset(zero, 0, sizeof zero);
-
- for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts)
+ /*-------------------------------------------------------------------------
+ * read/write
+ *-------------------------------------------------------------------------
+ */
+ if (nelmts)
{
- /* calculate the hyperslab size */
- if (rank > 0)
+ size_t need = (size_t)(nelmts*msize); /* bytes needed */
+ if ( need < H5TOOLS_MALLOCSIZE )
+ buf = HDmalloc(need);
+
+ if (buf != NULL )
{
- for (i = 0, hs_nelmts = 1; i < rank; i++)
- {
- hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]);
- hs_nelmts *= hs_size[i];
- }
-
- if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0)
+ if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
goto error;
- if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0)
+ if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
goto error;
- }
- else
- {
- H5Sselect_all(f_space_id);
- H5Sselect_all(sm_space);
- hs_nelmts = 1;
- } /* rank */
-
- /* read/write */
- if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
- goto error;
- if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
- goto error;
+ }
- /* reclaim any VL memory, if necessary */
- if(vl_data)
- H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf);
-
- /* calculate the next hyperslab offset */
- for (i = rank, carry = 1; i > 0 && carry; --i)
+ else /* possibly not enough memory, read/write by hyperslabs */
+
{
- hs_offset[i - 1] += hs_size[i - 1];
- if (hs_offset[i - 1] == dims[i - 1])
- hs_offset[i - 1] = 0;
- else
- carry = 0;
- } /* i */
- } /* elmtno */
-
- H5Sclose(sm_space);
- /* free */
- if (sm_buf!=NULL)
- {
- HDfree(sm_buf);
- sm_buf=NULL;
- }
- } /* hyperslab read */
+ size_t p_type_nbytes = msize; /*size of memory type */
+ hsize_t p_nelmts = nelmts; /*total selected elmts */
+ hsize_t elmtno; /*counter */
+ int carry; /*counter carry value */
+ unsigned int vl_data = 0; /*contains VL datatypes */
+
+ /* stripmine info */
+ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
+ hsize_t sm_nbytes; /*bytes per stripmine */
+ hsize_t sm_nelmts; /*elements per stripmine*/
+ hid_t sm_space; /*stripmine data space */
+
+ /* hyperslab info */
+ hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */
+ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */
+ hsize_t hs_nelmts; /*elements in request */
+ hsize_t zero[8]; /*vector of zeros */
+
+ /* check if we have VL data in the dataset's datatype */
+ if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE)
+ vl_data = TRUE;
+
+ /*
+ * determine the strip mine size and allocate a buffer. The strip mine is
+ * a hyperslab whose size is manageable.
+ */
+ sm_nbytes = p_type_nbytes;
+
+ for (i = rank; i > 0; --i) {
+ sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+ sm_nbytes *= sm_size[i - 1];
+ assert(sm_nbytes > 0);
+ }
+ sm_buf = HDmalloc((size_t)sm_nbytes);
+
+ sm_nelmts = sm_nbytes / p_type_nbytes;
+ sm_space = H5Screate_simple(1, &sm_nelmts, NULL);
+
+ /* the stripmine loop */
+ memset(hs_offset, 0, sizeof hs_offset);
+ memset(zero, 0, sizeof zero);
+
+ for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts)
+ {
+ /* calculate the hyperslab size */
+ if (rank > 0)
+ {
+ for (i = 0, hs_nelmts = 1; i < rank; i++)
+ {
+ hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]);
+ hs_nelmts *= hs_size[i];
+ }
+
+ if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0)
+ goto error;
+ if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0)
+ goto error;
+ }
+ else
+ {
+ H5Sselect_all(f_space_id);
+ H5Sselect_all(sm_space);
+ hs_nelmts = 1;
+ } /* rank */
+
+ /* read/write */
+ if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
+ goto error;
+ if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
+ goto error;
+
+ /* reclaim any VL memory, if necessary */
+ if(vl_data)
+ H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf);
+
+ /* calculate the next hyperslab offset */
+ for (i = rank, carry = 1; i > 0 && carry; --i)
+ {
+ hs_offset[i - 1] += hs_size[i - 1];
+ if (hs_offset[i - 1] == dims[i - 1])
+ hs_offset[i - 1] = 0;
+ else
+ carry = 0;
+ } /* i */
+ } /* elmtno */
+
+ H5Sclose(sm_space);
+ /* free */
+ if (sm_buf!=NULL)
+ {
+ HDfree(sm_buf);
+ sm_buf=NULL;
+ }
+ } /* hyperslab read */
}/*nelmts*/
-
- /*-------------------------------------------------------------------------
- * amount of compression used
- *-------------------------------------------------------------------------
- */
- if (options->verbose)
- {
- if (apply_s && apply_f)
+
+ /*-------------------------------------------------------------------------
+ * amount of compression used
+ *-------------------------------------------------------------------------
+ */
+ if (options->verbose)
{
- /* get the storage size of the input dataset */
- dsize_out=H5Dget_storage_size(dset_out);
- PER((hssize_t)dsize_in,(hssize_t)dsize_out);
- print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0);
+ if (apply_s && apply_f)
+ {
+ /* get the storage size of the input dataset */
+ dsize_out=H5Dget_storage_size(dset_out);
+ PER((hssize_t)dsize_in,(hssize_t)dsize_out);
+ print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0);
+ }
+ else
+ print_dataset_info(dcpl_id,travt->objs[i].name,0);
}
- else
- print_dataset_info(dcpl_id,travt->objs[i].name,0);
- }
-
- if (apply_s==0 && options->verbose)
- printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n",
- travt->objs[i].name,
- (int)options->threshold);
-
- if (apply_f==0 && options->verbose)
+
+ if (apply_s==0 && options->verbose)
+ printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n",
+ travt->objs[i].name,
+ (int)options->threshold);
+
+ if (apply_f==0 && options->verbose)
printf(" <warning: could not apply the filter to %s>\n",
travt->objs[i].name);
-
- /*-------------------------------------------------------------------------
- * copy attrs
- *-------------------------------------------------------------------------
- */
- if (copy_attr(dset_in,dset_out,options)<0)
- goto error;
-
- /*close */
- if (H5Dclose(dset_out)<0)
- goto error;
-
+
+ /*-------------------------------------------------------------------------
+ * copy attrs
+ *-------------------------------------------------------------------------
+ */
+ if (copy_attr(dset_in,dset_out,options)<0)
+ goto error;
+
+ /*close */
+ if (H5Dclose(dset_out)<0)
+ goto error;
+
}/*H5T_STD_REF_OBJ*/
}/*can_read*/
+
+
+ /*-------------------------------------------------------------------------
+ * close
+ *-------------------------------------------------------------------------
+ */
+ if (H5Tclose(ftype_id)<0)
+ goto error;
+ if (H5Tclose(wtype_id)<0)
+ goto error;
+ if (H5Pclose(dcpl_id)<0)
+ goto error;
+ if (H5Pclose(dcpl_out)<0)
+ goto error;
+ if (H5Sclose(f_space_id)<0)
+ goto error;
+ if (H5Dclose(dset_in)<0)
+ goto error;
+ }
+ /*-------------------------------------------------------------------------
+ * we do not have request for filter/chunking use H5Ocopy instead
+ *-------------------------------------------------------------------------
+ */
+ else
+ {
+ hid_t pid;
+
+ /* create property to pass copy options */
+ if ( (pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+ goto error;
+
+ /*-------------------------------------------------------------------------
+ * do the copy
+ *-------------------------------------------------------------------------
+ */
+
+ if (H5Ocopy(fidin, /* Source file or group identifier */
+ travt->objs[i].name, /* Name of the source object to be copied */
+ fidout, /* Destination file or group identifier */
+ travt->objs[i].name, /* Name of the destination object */
+ pid, /* Properties which apply to the copy */
+ H5P_DEFAULT)<0) /* Properties which apply to the new hard link */
+ goto error;
+
+ /* close property */
+ if (H5Pclose(pid)<0)
+ goto error;
+
+ } /* end do we have request for filter/chunking */
-/*-------------------------------------------------------------------------
- * close
- *-------------------------------------------------------------------------
- */
- if (H5Tclose(ftype_id)<0)
- goto error;
- if (H5Tclose(wtype_id)<0)
- goto error;
- if (H5Pclose(dcpl_id)<0)
- goto error;
- if (H5Pclose(dcpl_out)<0)
- goto error;
- if (H5Sclose(f_space_id)<0)
- goto error;
- if (H5Dclose(dset_in)<0)
- goto error;
break;
@@ -686,8 +755,6 @@ int do_copy_objects(hid_t fidin,
/*-------------------------------------------------------------------------
* the root is a special case, we get an ID for the root group
* and copy its attributes using that ID
- * it must be done last, because the attributes might contain references to
- * objects in the object list
*-------------------------------------------------------------------------
*/
@@ -827,8 +894,6 @@ int copy_attr(hid_t loc_in,
*/
if ( ! H5Tequal(wtype_id, H5T_STD_REF_OBJ))
{
-
-
/*-------------------------------------------------------------------------
* read to memory
*-------------------------------------------------------------------------
@@ -894,6 +959,73 @@ error:
return -1;
}
+/*-------------------------------------------------------------------------
+ * Function: do_hardlinks
+ *
+ * Purpose: duplicate hard links
+ *
+ * Return: 0, ok, -1 no
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: December, 10, 2003
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static
+int do_hardlinks(hid_t fidout,trav_table_t *travt)
+{
+ int i, j;
+
+ for ( i = 0; i < travt->nobjs; i++)
+ {
+ switch ( travt->objs[i].type )
+ {
+ case H5G_GROUP:
+
+ if (travt->objs[i].nlinks)
+ {
+ for ( j=0; j<travt->objs[i].nlinks; j++)
+ {
+ if (H5Glink(fidout,
+ H5L_TYPE_HARD,
+ travt->objs[i].name,
+ travt->objs[i].links[j].new_name)<0)
+ return -1;
+ }
+ }
+
+ break;
+
+ case H5G_DATASET:
+ if (travt->objs[i].nlinks)
+ {
+ for ( j=0; j<travt->objs[i].nlinks; j++){
+ if (H5Glink(fidout,
+ H5L_TYPE_HARD,
+ travt->objs[i].name,
+ travt->objs[i].links[j].new_name)<0)
+ return -1;
+ }
+ }
+ break;
+ case H5G_TYPE:
+ case H5G_LINK:
+ case H5G_UDLINK:
+
+ /*nothing to do */
+ break;
+
+ default:
+
+ break;
+ }
+ }
+
+ return 0;
+
+}
generated by cgit v0.12 at 2024-08-27 16:26:10 (GMT)