summaryrefslogtreecommitdiffstats
path: root/tools/lib
diff options
context:
space:
mode:
authorPedro Vicente Nunes <pvn@hdfgroup.org>2006-10-19 17:07:26 (GMT)
committerPedro Vicente Nunes <pvn@hdfgroup.org>2006-10-19 17:07:26 (GMT)
commita1edc7069162a619957c7407d04b91e081044219 (patch)
treeca5e64791fcad2730a6ae92d7ba667bf9306cc41 /tools/lib
parent78a3d673e6cb491d8087b74d0e2cbdd8ca89bb47 (diff)
downloadhdf5-a1edc7069162a619957c7407d04b91e081044219.zip
hdf5-a1edc7069162a619957c7407d04b91e081044219.tar.gz
hdf5-a1edc7069162a619957c7407d04b91e081044219.tar.bz2
[svn-r12784]
Fixes for bugs 676, 228 676: both h5repack and h5diff use H5Dread. In the case of a "big" dataset, use read/write by hyperslabs the same way h5dump uses. An arbitrary value of 1GB was defined for "big", i.e, if the dataset is greater than 1GB, then read/write by hyperslabs 228: use the file type in read/write by default. A new switch -n was introduced if the user wants to use a native type, which was the previous use by default. Added a new test for h5repack that repacks a 1GB dataset Tested: heping (serial, parallel), sol, copper
Diffstat (limited to 'tools/lib')
-rw-r--r--tools/lib/h5diff_dset.c336
-rw-r--r--tools/lib/h5tools.c11
-rw-r--r--tools/lib/h5tools.h17
3 files changed, 261 insertions, 103 deletions
diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c
index 012552f..630b408 100644
--- a/tools/lib/h5diff_dset.c
+++ b/tools/lib/h5diff_dset.c
@@ -191,33 +191,36 @@ hsize_t diff_datasetid( hid_t did1,
const char *obj2_name,
diff_opt_t *options )
{
- hid_t sid1;
- hid_t sid2;
- hid_t f_tid1;
- hid_t f_tid2; /* file data type */
- hid_t m_tid1;
- hid_t m_tid2; /* memory data type */
- size_t m_size1;
- size_t m_size2; /* size of type in memory */
- H5T_sign_t sign1;
- H5T_sign_t sign2; /* sign of type */
- int rank1;
- int rank2;
- void *buf1=NULL;
- void *buf2=NULL;
- hsize_t nelmts1;
- hsize_t nelmts2;
- hsize_t dims1[H5S_MAX_RANK];
- hsize_t dims2[H5S_MAX_RANK];
- hsize_t maxdim1[H5S_MAX_RANK];
- hsize_t maxdim2[H5S_MAX_RANK];
- const char *name1=NULL; /* relative names */
- const char *name2=NULL;
- hsize_t storage_size1;
- hsize_t storage_size2;
- hsize_t nfound=0; /* number of differences found */
- int cmp=1; /* do diff or not */
- int i;
+ hid_t sid1;
+ hid_t sid2;
+ hid_t f_tid1;
+ hid_t f_tid2;
+ hid_t m_tid1;
+ hid_t m_tid2;
+ size_t m_size1;
+ size_t m_size2;
+ H5T_sign_t sign1;
+ H5T_sign_t sign2;
+ int rank1;
+ int rank2;
+ hsize_t nelmts1;
+ hsize_t nelmts2;
+ hsize_t dims1[H5S_MAX_RANK];
+ hsize_t dims2[H5S_MAX_RANK];
+ hsize_t maxdim1[H5S_MAX_RANK];
+ hsize_t maxdim2[H5S_MAX_RANK];
+ const char *name1=NULL; /* relative names */
+ const char *name2=NULL;
+ hsize_t storage_size1;
+ hsize_t storage_size2;
+ hsize_t nfound=0; /* number of differences found */
+ int cmp=1; /* do diff or not */
+ void *buf1=NULL;
+ void *buf2=NULL;
+ void *sm_buf1=NULL;
+ void *sm_buf2=NULL;
+ size_t need; /* bytes needed for malloc */
+ int i;
/* Get the dataspace handle */
if ( (sid1 = H5Dget_space(did1)) < 0 )
@@ -272,23 +275,22 @@ hsize_t diff_datasetid( hid_t did1,
options->not_cmp=1;
}
-
/*-------------------------------------------------------------------------
* check for comparable TYPE and SPACE
*-------------------------------------------------------------------------
*/
if (diff_can_type(f_tid1,
- f_tid2,
- rank1,
- rank2,
- dims1,
- dims2,
- maxdim1,
- maxdim2,
- obj1_name,
- obj2_name,
- options)!=1)
+ f_tid2,
+ rank1,
+ rank2,
+ dims1,
+ dims2,
+ maxdim1,
+ maxdim2,
+ obj1_name,
+ obj2_name,
+ options)!=1)
{
cmp=0;
options->not_cmp=1;
@@ -307,11 +309,6 @@ hsize_t diff_datasetid( hid_t did1,
m_size1 = H5Tget_size( m_tid1 );
m_size2 = H5Tget_size( m_tid2 );
-#if defined (H5DIFF_DEBUG)
- if (obj1_name)
- print_sizes(obj1_name,obj2_name,f_tid1,f_tid2,m_tid1,m_tid2);
-#endif
-
/*-------------------------------------------------------------------------
* check for different signed/unsigned types
*-------------------------------------------------------------------------
@@ -353,7 +350,6 @@ hsize_t diff_datasetid( hid_t did1,
nelmts2 *= dims2[i];
}
- /* only assert if the space is the same */
assert(nelmts1==nelmts2);
/*-------------------------------------------------------------------------
@@ -381,56 +377,176 @@ hsize_t diff_datasetid( hid_t did1,
m_size2 = H5Tget_size( m_tid2 );
}
-#if defined (H5DIFF_DEBUG)
- printf("WARNING: Size was upgraded\n");
- if (obj1_name)
- print_sizes(obj1_name,obj2_name,f_tid1,f_tid2,m_tid1,m_tid2);
-#endif
}
assert(m_size1==m_size2);
-
- buf1 = (void *) HDmalloc((unsigned) (nelmts1*m_size1));
- buf2 = (void *) HDmalloc((unsigned) (nelmts2*m_size2));
-
- if ( buf1 == NULL || buf2 == NULL )
- {
- printf( "cannot read into memory\n" );
- goto error;
+ /* print names */
+ if (obj1_name) {
+ name1=diff_basename(obj1_name);
+ }
+ if (obj2_name) {
+ name2=diff_basename(obj2_name);
}
+
/*-------------------------------------------------------------------------
- * read
+ * read/compare
*-------------------------------------------------------------------------
*/
- if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 )
- goto error;
-
- if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 )
- goto error;
-
-/*-------------------------------------------------------------------------
- * array compare
- *-------------------------------------------------------------------------
- */
- if (obj1_name) {
- name1=diff_basename(obj1_name);
+ need = (size_t)(nelmts1*m_size1); /* bytes needed */
+ if ( need < H5TOOLS_MALLOCSIZE )
+ {
+ buf1 = HDmalloc(need);
+ buf2 = HDmalloc(need);
}
- if (obj2_name) {
- name2=diff_basename(obj2_name);
+
+ if ( buf1!=NULL && buf2!=NULL)
+ {
+ if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 )
+ goto error;
+ if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 )
+ goto error;
+
+ /* array diff */
+ nfound = diff_array(buf1,
+ buf2,
+ nelmts1,
+ rank1,
+ dims1,
+ options,
+ name1,
+ name2,
+ m_tid1,
+ did1,
+ did2);
}
- nfound = diff_array(buf1,
- buf2,
- nelmts1,
- rank1,
- dims1,
- options,
- name1,
- name2,
- m_tid1,
- did1,
- did2);
+
+ else /* possibly not enough memory, read/compare by hyperslabs */
+
+ {
+ size_t p_type_nbytes = m_size1; /*size of memory type */
+ hsize_t p_nelmts = nelmts1; /*total selected elmts */
+ hsize_t elmtno; /*counter */
+ int carry; /*counter carry value */
+ unsigned int vl_data = 0; /*contains VL datatypes */
+
+ /* stripmine info */
+ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
+ hsize_t sm_nbytes; /*bytes per stripmine */
+ hsize_t sm_nelmts; /*elements per stripmine*/
+ hid_t sm_space; /*stripmine data space */
+
+ /* hyperslab info */
+ hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */
+ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */
+ unsigned char *sm_buf1 = NULL; /*buffer for raw data */
+ unsigned char *sm_buf2 = NULL; /*buffer for raw data */
+ hsize_t hs_nelmts; /*elements in request */
+ hsize_t zero[8]; /*vector of zeros */
+
+ /* check if we have VL data in the dataset's datatype */
+ if (H5Tdetect_class(m_tid1, H5T_VLEN) == TRUE)
+ vl_data = TRUE;
+
+ /*
+ * determine the strip mine size and allocate a buffer. The strip mine is
+ * a hyperslab whose size is manageable.
+ */
+ sm_nbytes = p_type_nbytes;
+
+ for (i = rank1; i > 0; --i) {
+ sm_size[i - 1] = MIN(dims1[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+ sm_nbytes *= sm_size[i - 1];
+ assert(sm_nbytes > 0);
+ }
+
+ sm_buf1 = malloc((size_t)sm_nbytes);
+ sm_buf2 = malloc((size_t)sm_nbytes);
+
+ sm_nelmts = sm_nbytes / p_type_nbytes;
+ sm_space = H5Screate_simple(1, &sm_nelmts, NULL);
+
+ /* the stripmine loop */
+ memset(hs_offset, 0, sizeof hs_offset);
+ memset(zero, 0, sizeof zero);
+
+ for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts)
+ {
+ /* calculate the hyperslab size */
+ if (rank1 > 0)
+ {
+ for (i = 0, hs_nelmts = 1; i < rank1; i++)
+ {
+ hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]);
+ hs_nelmts *= hs_size[i];
+ }
+ if (H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0)
+ goto error;
+ if (H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0)
+ goto error;
+ if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0)
+ goto error;
+ }
+ else
+ {
+ H5Sselect_all(sid1);
+ H5Sselect_all(sid2);
+ H5Sselect_all(sm_space);
+ hs_nelmts = 1;
+ } /* rank */
+
+ if ( H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0 )
+ goto error;
+ if ( H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0 )
+ goto error;
+
+ /* array diff */
+ nfound = diff_array(sm_buf1,
+ sm_buf2,
+ hs_nelmts,
+ rank1,
+ dims1,
+ options,
+ name1,
+ name2,
+ m_tid1,
+ did1,
+ did2);
+
+ /* reclaim any VL memory, if necessary */
+ if(vl_data)
+ {
+ H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1);
+ H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2);
+ }
+
+ /* calculate the next hyperslab offset */
+ for (i = rank1, carry = 1; i > 0 && carry; --i)
+ {
+ hs_offset[i - 1] += hs_size[i - 1];
+ if (hs_offset[i - 1] == dims1[i - 1])
+ hs_offset[i - 1] = 0;
+ else
+ carry = 0;
+ } /* i */
+ } /* elmtno */
+
+ H5Sclose(sm_space);
+ /* free */
+ if (sm_buf1!=NULL)
+ {
+ free(sm_buf1);
+ sm_buf1=NULL;
+ }
+ if (sm_buf2!=NULL)
+ {
+ free(sm_buf2);
+ sm_buf2=NULL;
+ }
+
+ } /* hyperslab read */
+
/*-------------------------------------------------------------------------
* compare attributes
* the if condition refers to cases when the dataset is a referenced object
@@ -446,12 +562,29 @@ hsize_t diff_datasetid( hid_t did1,
* close
*-------------------------------------------------------------------------
*/
+
+ /* free */
+ if (buf1!=NULL)
+ {
+ free(buf1);
+ buf1=NULL;
+ }
+ if (buf2!=NULL)
+ {
+ free(buf2);
+ buf2=NULL;
+ }
+ if (sm_buf1!=NULL)
+ {
+ free(sm_buf1);
+ sm_buf1=NULL;
+ }
+ if (sm_buf2!=NULL)
+ {
+ free(sm_buf2);
+ sm_buf2=NULL;
+ }
-
- if ( buf1) HDfree(buf1);
- if ( buf2) HDfree(buf2);
- /* close */
- /* disable error reporting */
H5E_BEGIN_TRY {
H5Sclose(sid1);
H5Sclose(sid2);
@@ -459,16 +592,35 @@ hsize_t diff_datasetid( hid_t did1,
H5Tclose(f_tid2);
H5Tclose(m_tid1);
H5Tclose(m_tid2);
- /* enable error reporting */
} H5E_END_TRY;
return nfound;
error:
options->err_stat=1;
- if ( buf1) HDfree(buf1);
- if ( buf2) HDfree(buf2);
- /* close */
+
+ /* free */
+ if (buf1!=NULL)
+ {
+ free(buf1);
+ buf1=NULL;
+ }
+ if (buf2!=NULL)
+ {
+ free(buf2);
+ buf2=NULL;
+ }
+ if (sm_buf1!=NULL)
+ {
+ free(sm_buf1);
+ sm_buf1=NULL;
+ }
+ if (sm_buf2!=NULL)
+ {
+ free(sm_buf2);
+ sm_buf2=NULL;
+ }
+
/* disable error reporting */
H5E_BEGIN_TRY {
H5Sclose(sid1);
diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
index 362b070..8279ec9 100644
--- a/tools/lib/h5tools.c
+++ b/tools/lib/h5tools.c
@@ -29,17 +29,6 @@
#include "h5tools_utils.h"
#include "H5private.h"
-/*
- * The output functions need a temporary buffer to hold a piece of the
- * dataset while it's being printed. This constant sets the limit on the
- * size of that temporary buffer in bytes. For efficiency's sake, choose the
- * largest value suitable for your machine (for testing use a small value).
- */
-#if 1
-#define H5TOOLS_BUFSIZE (1024 * 1024)
-#else
-#define H5TOOLS_BUFSIZE (1024)
-#endif /* 1 */
#define ALIGN(A,Z) ((((A) + (Z) - 1) / (Z)) * (Z))
diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h
index 357154b..70d8734 100644
--- a/tools/lib/h5tools.h
+++ b/tools/lib/h5tools.h
@@ -30,6 +30,23 @@
#define END_OF_DATA 0x0002
/*
+ * The output functions need a temporary buffer to hold a piece of the
+ * dataset while it's being printed. This constant sets the limit on the
+ * size of that temporary buffer in bytes. For efficiency's sake, choose the
+ * largest value suitable for your machine (for testing use a small value).
+ */
+#if 1
+#define H5TOOLS_BUFSIZE (1024 * 1024)
+#else
+#define H5TOOLS_BUFSIZE (1024)
+#endif
+
+/*
+ * Maximum size used in a call to malloc
+ */
+#define H5TOOLS_MALLOCSIZE ((size_t)1024 * 1024 * 1024)
+
+/*
* Information about how to format output.
*/
typedef struct h5tool_format_t {