summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--tools/h5repack/h5repack_copy.c339
1 files changed, 178 insertions, 161 deletions
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index b0d475f..999dce4 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -187,11 +187,11 @@ int copy_objects(const char* fnamein,
} /* end if */
} /* end if */
} /* end if */
+
+
-
-
-#if defined (H5REPACK_DEBUG)
+#if defined (H5REPACK_DEBUG)
print_user_block(fnamein,fidin);
#endif
@@ -207,35 +207,35 @@ int copy_objects(const char* fnamein,
if(fcpl != H5P_DEFAULT)
{
/* set user block size */
- if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
+ if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
{
error_msg(progname, "failed to set userblock size\n");
goto out;
- }
-
+ }
+
}
-
+
else
{
-
+
/* create a file creation property list */
- if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
{
error_msg(progname, "fail to create a file creation property list\n");
goto out;
- }
-
+ }
+
/* set user block size */
- if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
+ if(H5Pset_userblock(fcpl, options->ublock_size) < 0)
{
error_msg(progname, "failed to set userblock size\n");
goto out;
- }
-
+ }
+
}
-
-
-
+
+
+
}
@@ -244,39 +244,39 @@ int copy_objects(const char* fnamein,
*-------------------------------------------------------------------------
*/
-
+
if ( options->alignment > 0 )
{
/* either use the FCPL already created or create a new one */
if (fapl != H5P_DEFAULT)
{
-
+
if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
{
error_msg(progname, "failed to set alignment\n");
goto out;
}
-
+
}
-
+
else
{
-
+
/* create a file access property list */
- if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
{
error_msg(progname, "Could not create file access property list\n");
goto out;
- }
-
+ }
+
if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
{
error_msg(progname, "failed to set alignment\n");
goto out;
}
-
- }
-
+
+ }
+
}
@@ -285,28 +285,33 @@ int copy_objects(const char* fnamein,
*-------------------------------------------------------------------------
*/
-
+
if(options->verbose)
printf("Making file <%s>...\n",fnameout);
-
-
- if((fidout = H5Fcreate(fnameout,H5F_ACC_TRUNC, fcpl, fapl)) < 0)
+
+
+ if((fidout = H5Fcreate(fnameout,H5F_ACC_TRUNC, fcpl, fapl)) < 0)
{
error_msg(progname, "<%s>: Could not create file\n", fnameout );
goto out;
- } /* end if */
-
+ }
+
/*-------------------------------------------------------------------------
* write a new user block if requested
*-------------------------------------------------------------------------
*/
if ( options->ublock_size > 0 )
- {
- copy_user_block( options->ublock_filename, fnameout, options->ublock_size);
+ {
+ if ( copy_user_block( options->ublock_filename, fnameout, options->ublock_size) < 0 )
+ {
+ error_msg(progname, "Could not copy user block. Exiting...\n");
+ goto out;
+
+ }
}
-
+
/* init table */
trav_table_init(&travt);
@@ -329,14 +334,11 @@ int copy_objects(const char* fnamein,
* and create hard links
*-------------------------------------------------------------------------
*/
- if(do_copy_refobjs(fidin, fidout, travt, options) < 0) {
+ if ( do_copy_refobjs(fidin, fidout, travt, options) < 0 )
+ {
printf("h5repack: <%s>: Could not copy data to: %s\n", fnamein, fnameout);
goto out;
- } /* end if */
-
- /* free table */
- trav_table_free(travt);
-
+ }
/*-------------------------------------------------------------------------
* close
@@ -351,10 +353,25 @@ int copy_objects(const char* fnamein,
H5Fclose(fidin);
H5Fclose(fidout);
+
+ /* free table */
+ trav_table_free(travt);
+ travt = NULL;
- /* write only the input file user block if there is no user block file input */
- if(ub_size > 0 && options->ublock_size == 0)
- copy_user_block(fnamein, fnameout, ub_size);
+ /*-------------------------------------------------------------------------
+ * write only the input file user block if there is no user block file input
+ *-------------------------------------------------------------------------
+ */
+
+ if( ub_size > 0 && options->ublock_size == 0 )
+ {
+ if ( copy_user_block(fnamein, fnameout, ub_size) < 0 )
+ {
+ error_msg(progname, "Could not copy user block. Exiting...\n");
+ goto out;
+
+ }
+ }
return 0;
@@ -387,7 +404,7 @@ out:
*
* Date: October, 23, 2003
*
- * Modifications:
+ * Modifications:
*
* July 2004: Introduced the extra EC or NN option for SZIP
*
@@ -399,47 +416,47 @@ out:
* October 2006: Read by hyperslabs for big datasets.
*
* A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done
- * i.e., if the memory needed to read a dataset is greater than this limit,
- * then hyperslab I/O is done instead of one operation I/O
+ * i.e., if the memory needed to read a dataset is greater than this limit,
+ * then hyperslab I/O is done instead of one operation I/O
* For each dataset, the memory needed is calculated according to
*
* memory needed = number of elements * size of each element
*
- * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations
+ * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations
* are done
*
* H5Dread( input_dataset1 )
* H5Dread( input_dataset2 )
*
- * with all elements in the datasets selected. If the memory needed is greater than
+ * with all elements in the datasets selected. If the memory needed is greater than
* H5TOOLS_MALLOCSIZE, then the following operations are done instead:
*
- * a strip mine is defined for each dimension k (a strip mine is defined as a
+ * a strip mine is defined for each dimension k (a strip mine is defined as a
* hyperslab whose size is memory manageable) according to the formula
*
* (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type)
*
- * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures
- * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip
- * mine size k is simply defined as its dimension k, but for larger datasets the
+ * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures
+ * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip
+ * mine size k is simply defined as its dimension k, but for larger datasets the
* hyperslab size is still memory manageable.
- * a cycle is done until the number of elements in the dataset is reached. In each
- * iteration, two parameters are defined for the function H5Sselect_hyperslab,
+ * a cycle is done until the number of elements in the dataset is reached. In each
+ * iteration, two parameters are defined for the function H5Sselect_hyperslab,
* the start and size of each hyperslab, according to
*
* (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k])
*
- * where hyperslab_offset [k] is initially set to zero, and later incremented in
- * hyperslab_size[k] offsets. The reason for the operation
+ * where hyperslab_offset [k] is initially set to zero, and later incremented in
+ * hyperslab_size[k] offsets. The reason for the operation
*
* dimension[k] - hyperslab_offset[k]
*
- * in (2) is that, when using the strip mine size, it assures that the "remaining" part
+ * in (2) is that, when using the strip mine size, it assures that the "remaining" part
* of the dataset that does not fill an entire strip mine is processed.
*
- * November 2006: Use H5Ocopy in the copy of objects. The logic for using
- * H5Ocopy or not is if a change of filters or layout is requested by the user
- * then use read/write else use H5Ocopy.
+ * November 2006: Use H5Ocopy in the copy of objects. The logic for using
+ * H5Ocopy or not is if a change of filters or layout is requested by the user
+ * then use read/write else use H5Ocopy.
*
* May, 1, 2008: Add a printing of the compression ratio of old size / new size
*
@@ -518,7 +535,7 @@ int do_copy_objects(hid_t fidin,
goto error;
if((grp_out = H5Gcreate2(fidout, travt->objs[i].name, H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
- goto error;
+ goto error;
}
else {
if((grp_out = H5Gcreate2(fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
@@ -565,7 +582,7 @@ int do_copy_objects(hid_t fidin,
goto error;
if(H5Dclose(dset_in) < 0)
goto error;
-
+
/*-------------------------------------------------------------------------
* check if we should use H5Ocopy or not
@@ -599,11 +616,11 @@ int do_copy_objects(hid_t fidin,
if(options->use_native == 1)
wtype_id = h5tools_get_native_type(ftype_id);
else
- wtype_id = H5Tcopy(ftype_id);
+ wtype_id = H5Tcopy(ftype_id);
if((msize = H5Tget_size(wtype_id)) == 0)
goto error;
-
+
/*-------------------------------------------------------------------------
* check if the dataset creation property list has filters that
* are not registered in the current configuration
@@ -647,7 +664,7 @@ int do_copy_objects(hid_t fidin,
dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT);
} H5E_END_TRY;
- if(dset_out == FAIL)
+ if(dset_out == FAIL)
{
if(options->verbose)
printf(" warning: could not create dataset <%s>. Applying original settings\n",
@@ -674,7 +691,7 @@ int do_copy_objects(hid_t fidin,
goto error;
if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf) < 0)
goto error;
- }
+ }
else /* possibly not enough memory, read/write by hyperslabs */
{
@@ -695,7 +712,7 @@ int do_copy_objects(hid_t fidin,
hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */
hsize_t hs_nelmts; /*elements in request */
hsize_t zero[8]; /*vector of zeros */
- int k;
+ int k;
/* check if we have VL data in the dataset's datatype */
if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE)
@@ -721,12 +738,12 @@ int do_copy_objects(hid_t fidin,
memset(hs_offset, 0, sizeof hs_offset);
memset(zero, 0, sizeof zero);
- for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts)
+ for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts)
{
/* calculate the hyperslab size */
- if (rank > 0)
+ if (rank > 0)
{
- for (k = 0, hs_nelmts = 1; k < rank; k++)
+ for (k = 0, hs_nelmts = 1; k < rank; k++)
{
hs_size[k] = MIN(dims[k] - hs_offset[k], sm_size[k]);
hs_nelmts *= hs_size[k];
@@ -736,8 +753,8 @@ int do_copy_objects(hid_t fidin,
goto error;
if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0)
goto error;
- }
- else
+ }
+ else
{
H5Sselect_all(f_space_id);
H5Sselect_all(sm_space);
@@ -745,9 +762,9 @@ int do_copy_objects(hid_t fidin,
} /* rank */
/* read/write */
- if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
+ if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
goto error;
- if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
+ if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0)
goto error;
/* reclaim any VL memory, if necessary */
@@ -755,7 +772,7 @@ int do_copy_objects(hid_t fidin,
H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf);
/* calculate the next hyperslab offset */
- for (k = rank, carry = 1; k > 0 && carry; --k)
+ for (k = rank, carry = 1; k > 0 && carry; --k)
{
hs_offset[k - 1] += hs_size[k - 1];
if (hs_offset[k - 1] == dims[k - 1])
@@ -779,7 +796,7 @@ int do_copy_objects(hid_t fidin,
* amount of compression used
*-------------------------------------------------------------------------
*/
- if (options->verbose)
+ if (options->verbose)
{
double ratio=0;
@@ -796,13 +813,13 @@ int do_copy_objects(hid_t fidin,
a = dsize_in; b = dsize_out;
if (b!=0)
ratio = (double) a / (double) b;
-
+
print_dataset_info(dcpl_out,travt->objs[i].name,ratio,1);
}
else
print_dataset_info(dcpl_id,travt->objs[i].name,ratio,0);
- /* print a message that the filter was not applied
+ /* print a message that the filter was not applied
(in case there was a filter)
*/
if ( has_filter && apply_s == 0 )
@@ -853,16 +870,16 @@ int do_copy_objects(hid_t fidin,
* we do not have request for filter/chunking use H5Ocopy instead
*-------------------------------------------------------------------------
*/
- else
+ else
{
hid_t pid;
/* create property to pass copy options */
- if ( (pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+ if ( (pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
goto error;
/* set options for object copy */
- if(H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
+ if(H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
goto error;
/*-------------------------------------------------------------------------
@@ -875,7 +892,7 @@ int do_copy_objects(hid_t fidin,
fidout, /* Destination file or group identifier */
travt->objs[i].name, /* Name of the destination object */
pid, /* Properties which apply to the copy */
- H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */
+ H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */
goto error;
/* close property */
@@ -901,7 +918,7 @@ int do_copy_objects(hid_t fidin,
if (options->verbose)
printf(FORMAT_OBJ,"dset",travt->objs[i].name );
-
+
} /* end do we have request for filter/chunking */
@@ -938,7 +955,7 @@ int do_copy_objects(hid_t fidin,
if(H5Tclose(type_out) < 0)
goto error;
-
+
break;
@@ -1042,52 +1059,52 @@ int copy_attr(hid_t loc_in,
H5O_info_t oinfo; /* object info */
int j;
unsigned u;
-
+
if(H5Oget_info(loc_in, &oinfo) < 0)
goto error;
-
+
/*-------------------------------------------------------------------------
* copy all attributes
*-------------------------------------------------------------------------
*/
-
+
for ( u = 0; u < (unsigned)oinfo.num_attrs; u++)
{
-
+
buf=NULL;
-
+
/* open attribute */
if((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)u, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
-
+
/* get name */
if (H5Aget_name( attr_id, (size_t)255, name ) < 0)
goto error;
-
+
/* get the file datatype */
if ((ftype_id = H5Aget_type( attr_id )) < 0 )
goto error;
-
+
/* get the dataspace handle */
if ((space_id = H5Aget_space( attr_id )) < 0 )
goto error;
-
+
/* get dimensions */
if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 )
goto error;
-
+
nelmts=1;
for (j=0; j<rank; j++)
nelmts*=dims[j];
-
+
if (options->use_native==1)
wtype_id = h5tools_get_native_type(ftype_id);
else
- wtype_id = H5Tcopy(ftype_id);
-
+ wtype_id = H5Tcopy(ftype_id);
+
if ((msize=H5Tget_size(wtype_id))==0)
goto error;
-
+
/*-------------------------------------------------------------------------
* object references are a special case
* we cannot just copy the buffers, but instead we recreate the reference
@@ -1095,18 +1112,18 @@ int copy_attr(hid_t loc_in,
* the referenced objects
*-------------------------------------------------------------------------
*/
-
+
if (H5T_REFERENCE==H5Tget_class(wtype_id))
{
;
}
- else
+ else
{
/*-------------------------------------------------------------------------
* read to memory
*-------------------------------------------------------------------------
*/
-
+
buf = (void *)HDmalloc((size_t)(nelmts * msize));
if(buf == NULL) {
error_msg(progname, "cannot read into memory\n" );
@@ -1114,46 +1131,46 @@ int copy_attr(hid_t loc_in,
}
if(H5Aread(attr_id, wtype_id, buf) < 0)
goto error;
-
+
/*-------------------------------------------------------------------------
* copy
*-------------------------------------------------------------------------
*/
-
+
if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if(H5Awrite(attr_out, wtype_id, buf) < 0)
goto error;
-
+
/*close*/
if(H5Aclose(attr_out) < 0)
goto error;
-
-
+
+
if(buf)
free(buf);
-
+
} /*H5T_REFERENCE*/
-
-
+
+
if(options->verbose)
printf(FORMAT_OBJ_ATTR, "attr", name);
-
+
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
-
+
if (H5Tclose(ftype_id) < 0) goto error;
if (H5Tclose(wtype_id) < 0) goto error;
if (H5Sclose(space_id) < 0) goto error;
if (H5Aclose(attr_id) < 0) goto error;
-
+
} /* u */
-
-
+
+
return 0;
-
+
error:
H5E_BEGIN_TRY {
H5Tclose(ftype_id);
@@ -1193,27 +1210,27 @@ static void print_dataset_info(hid_t dcpl_id,
size_t cd_nelmts; /* filter client number of values */
char f_objname[256]; /* filter objname */
int i;
-
-
+
+
strcpy(strfilter,"\0");
-
+
/* get information about input filters */
if((nfilters = H5Pget_nfilters(dcpl_id)) < 0)
return;
-
+
for(i = 0; i < nfilters; i++) {
cd_nelmts = NELMTS(cd_values);
-
+
filtn = H5Pget_filter2(dcpl_id, (unsigned)i, &filt_flags, &cd_nelmts,
cd_values, sizeof(f_objname), f_objname, NULL);
-
+
switch(filtn) {
default:
break;
-
+
case H5Z_FILTER_DEFLATE:
strcat(strfilter,"GZIP ");
-
+
#if defined (PRINT_DEBUG)
{
unsigned level=cd_values[0];
@@ -1222,10 +1239,10 @@ static void print_dataset_info(hid_t dcpl_id,
}
#endif
break;
-
+
case H5Z_FILTER_SZIP:
strcat(strfilter,"SZIP ");
-
+
#if defined (PRINT_DEBUG)
{
unsigned options_mask=cd_values[0]; /* from dcpl, not filt*/
@@ -1238,29 +1255,29 @@ static void print_dataset_info(hid_t dcpl_id,
strcpy(temp,"NN) ");
}
strcat(strfilter,temp);
-
+
#endif
-
+
break;
-
+
case H5Z_FILTER_SHUFFLE:
strcat(strfilter,"SHUF ");
break;
-
+
case H5Z_FILTER_FLETCHER32:
strcat(strfilter,"FLET ");
break;
-
+
case H5Z_FILTER_NBIT:
strcat(strfilter,"NBIT ");
break;
-
+
case H5Z_FILTER_SCALEOFFSET:
strcat(strfilter,"SCALEOFFSET ");
break;
} /* switch */
}/*i*/
-
+
if(!pr)
printf(FORMAT_OBJ,"dset",objname );
else
@@ -1275,13 +1292,13 @@ static void print_dataset_info(hid_t dcpl_id,
}
/*-------------------------------------------------------------------------
- * Function: copy_user_block
+ * Function: copy_user_block
*
* Purpose: copy user block from one file to another
*
* Return: 0, ok, -1 no
*
- * Programmer: Peter Cao
+ * Programmer: Peter Cao
*
* Date: October, 25, 2007
*
@@ -1355,25 +1372,25 @@ done:
if(outfid > 0)
HDclose(outfid);
- return status;
+ return status;
}
/*-------------------------------------------------------------------------
- * Function: print_user_block
+ * Function: print_user_block
*
* Purpose: print user block
*
* Return: 0, ok, -1 no
*
- * Programmer: Pedro Vicente
+ * Programmer: Pedro Vicente
*
* Date: August, 20, 2008
*
*-------------------------------------------------------------------------
*/
-static
+static
void print_user_block(const char *filename, hid_t fid)
{
int fh; /* file handle */
@@ -1382,35 +1399,35 @@ void print_user_block(const char *filename, hid_t fid)
hid_t fcpl; /* file creation property list ID for HDF5 file */
int i;
- /* get user block size */
- if(( fcpl = H5Fget_create_plist(fid)) < 0)
+ /* get user block size */
+ if(( fcpl = H5Fget_create_plist(fid)) < 0)
{
error_msg(progname, "failed to retrieve file creation property list\n");
goto done;
- }
-
- if(H5Pget_userblock(fcpl, &ub_size) < 0)
+ }
+
+ if(H5Pget_userblock(fcpl, &ub_size) < 0)
{
error_msg(progname, "failed to retrieve userblock size\n");
goto done;
- }
-
- if(H5Pclose(fcpl) < 0)
+ }
+
+ if(H5Pclose(fcpl) < 0)
{
error_msg(progname, "failed to close property list\n");
goto done;
- }
-
+ }
+
/* open file */
- if((fh = HDopen(filename, O_RDONLY, 0)) < 0)
+ if((fh = HDopen(filename, O_RDONLY, 0)) < 0)
{
goto done;
}
-
+
size = ub_size;
/* read file */
- while(size > 0)
+ while(size > 0)
{
ssize_t nread; /* # of bytes read */
char rbuf[USERBLOCK_XFER_SIZE]; /* buffer for reading */
@@ -1421,7 +1438,7 @@ void print_user_block(const char *filename, hid_t fid)
else
nread = HDread(fh, rbuf, (size_t)size);
- for(i = 0; i < nread; i++)
+ for(i = 0; i < nread; i++)
{
printf("%c ", rbuf[i]);
@@ -1429,20 +1446,20 @@ void print_user_block(const char *filename, hid_t fid)
}
printf("\n");
- if(nread < 0)
+ if(nread < 0)
{
goto done;
- }
-
+ }
+
/* update size of userblock left to transfer */
size -= nread;
- }
+ }
done:
if(fh > 0)
HDclose(fh);
+
-
- return;
+ return;
}