summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobb Matzke <matzke@llnl.gov>1998-04-21 20:32:07 (GMT)
committerRobb Matzke <matzke@llnl.gov>1998-04-21 20:32:07 (GMT)
commit91a34f543da20aee0de3ad2ec2cc58f86750bcf6 (patch)
treed538ec3a0fba6996818c33ee57661bb1cc4ce209
parent54256b22c0fb6a38d1b33e531ef617b00c013ea6 (diff)
downloadhdf5-91a34f543da20aee0de3ad2ec2cc58f86750bcf6.zip
hdf5-91a34f543da20aee0de3ad2ec2cc58f86750bcf6.tar.gz
hdf5-91a34f543da20aee0de3ad2ec2cc58f86750bcf6.tar.bz2
[svn-r355] Changes since 19980417
---------------------- This checkin completes the compression stuff almost two weeks ahead of schedule. It should now all be working and documented. ./Makefile.in ./config/commence.in A distclean causes the distribution makefile to be copied to Makefile again. ./src/H5D.c Opening a compressed dataset now works properly. ./src/H5P.c ./src/H5Ppublic.h The H5Pset_compression() and H5Pget_compression() now work as documented. ./src/H5T.c Fixed a bug that causes the library to crash sometimes if debugging is turned on for the `t' package. ./src/H5Z.c Fixed a bug where the number of bytes attributed to compression overruns was too low in the debugging output. ./test/dsets.c More compression tests: reading of uninitialized data, opening an existing dataset and reading it, partial I/O which isn't aligned on chunk boundaries, and use of application-defined compression methods. ./MANIFEST Removed ./test/dspace.c.
-rw-r--r--MANIFEST1
-rw-r--r--Makefile.in1
-rw-r--r--config/commence.in1
-rw-r--r--src/H5D.c8
-rw-r--r--src/H5P.c33
-rw-r--r--src/H5Ppublic.h8
-rw-r--r--src/H5T.c4
-rw-r--r--src/H5Z.c2
-rw-r--r--test/dsets.c249
9 files changed, 267 insertions, 40 deletions
diff --git a/MANIFEST b/MANIFEST
index b4cafd4..7875183 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -195,7 +195,6 @@
./test/big.c
./test/cmpd_dset.c
./test/dsets.c
-./test/dspace.c
./test/dtypes.c
./test/extend.c
./test/external.c
diff --git a/Makefile.in b/Makefile.in
index 9b7ef97..0ab8db6 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -81,6 +81,7 @@ distclean:
$(RM) config/commence config/conclude config/depend
$(RM) config.cache config.log config.status src/H5config.h
$(RM) Makefile
+ $(CP) Makefile.dist Makefile
maintainer-clean:
@echo "This command is intented for maintainers to use;"
diff --git a/config/commence.in b/config/commence.in
index c85da9b..7b41932 100644
--- a/config/commence.in
+++ b/config/commence.in
@@ -18,6 +18,7 @@ LIBS=@LIBS@
AR=@AR@
RANLIB=@RANLIB@
RM=rm -f
+CP=cp
INSTALL=@INSTALL@
INSTALL_PROGRAM=@INSTALL_PROGRAM@
INSTALL_DATA=@INSTALL_DATA@
diff --git a/src/H5D.c b/src/H5D.c
index a49727a..45ed334 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -969,6 +969,14 @@ H5D_open(H5G_t *loc, const char *name)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
"unable to load type or space info from dataset header");
}
+
+ /* Get the optional compression message */
+ if (NULL==H5O_read (&(dataset->ent), H5O_COMPRESS, 0,
+ &(dataset->create_parms->compress))) {
+ H5E_clear ();
+ HDmemset (&(dataset->create_parms->compress), 0,
+ sizeof(dataset->create_parms->compress));
+ }
/*
* Get the raw data layout info. It's actually stored in two locations:
diff --git a/src/H5P.c b/src/H5P.c
index b141853..3c802e4 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -1859,6 +1859,10 @@ H5Pget_preserve (hid_t plist_id)
* specific compression initialization functions like
* H5Pset_deflate().
*
+ * The FLAGS, CD_SIZE, and CLIENT_DATA are copied to the
+ * property list and eventually to the file and passed to the
+ * compression functions.
+ *
* Return: Success: SUCCEED
*
* Failure: FAIL
@@ -1871,7 +1875,8 @@ H5Pget_preserve (hid_t plist_id)
*-------------------------------------------------------------------------
*/
herr_t
-H5Pset_compression (hid_t plist_id, H5Z_method_t method)
+H5Pset_compression (hid_t plist_id, H5Z_method_t method, unsigned int flags,
+ size_t cd_size, const void *client_data)
{
H5D_create_t *plist = NULL;
@@ -1891,7 +1896,12 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method)
/* Clear any previous compression method info, then set new value */
H5O_reset (H5O_COMPRESS, &(plist->compress));
plist->compress.method = method;
-
+ plist->compress.flags = flags;
+ plist->compress.cd_size = cd_size;
+ if (cd_size) {
+ plist->compress.client_data = H5MM_xmalloc (cd_size);
+ HDmemcpy (plist->compress.client_data, client_data, cd_size);
+ }
FUNC_LEAVE (SUCCEED);
}
@@ -1900,7 +1910,11 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method)
* Function: H5Pget_compression
*
* Purpose: Gets the compression method information from a dataset
- * creation property list.
+ * creation property list. The CLIENT_DATA buffer is initially
+ * CD_SIZE bytes. On return, CLIENT_DATA will be initialized
+ * with at most that many bytes, and CD_SIZE will contain the
+ * actual size of the client data, which might be larger than
+ * its original value.
*
* Return: Success: Compression method.
*
@@ -1914,7 +1928,8 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method)
*-------------------------------------------------------------------------
*/
H5Z_method_t
-H5Pget_compression (hid_t plist_id)
+H5Pget_compression (hid_t plist_id, unsigned int *flags/*out*/,
+ size_t *cd_size/*in,out*/, void *client_data/*out*/)
{
H5D_create_t *plist = NULL;
@@ -1927,6 +1942,16 @@ H5Pget_compression (hid_t plist_id)
"not a dataset creation property list");
}
+ /* Output values */
+ if (flags) *flags = plist->compress.flags;
+ if (cd_size) {
+ if (*cd_size>0 && client_data) {
+ HDmemcpy (client_data, plist->compress.client_data,
+ MIN(plist->compress.cd_size, *cd_size));
+ }
+ *cd_size = plist->compress.cd_size;
+ }
+
FUNC_LEAVE (plist->compress.method);
}
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index bd87342..503f8c5 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -88,8 +88,12 @@ herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
herr_t H5Pset_preserve (hid_t plist_id, hbool_t status);
int H5Pget_preserve (hid_t plist_id);
-herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method);
-H5Z_method_t H5Pget_compression (hid_t plist_id);
+herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method,
+ unsigned int flags, size_t cd_size,
+ const void *client_data);
+H5Z_method_t H5Pget_compression (hid_t plist_id, unsigned int *flags/*out*/,
+ size_t *cd_size/*in,out*/,
+ void *client_data/*out*/);
herr_t H5Pset_deflate (hid_t plist_id, int level);
int H5Pget_deflate (hid_t plist_id);
diff --git a/src/H5T.c b/src/H5T.c
index 488e30d..0ca0377 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -3424,6 +3424,10 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg,
FUNC_ENTER(H5T_find, NULL);
+ if (!noop_cdata.stats) {
+ noop_cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t));
+ }
+
/* No-op case */
if (need_bkg<H5T_BKG_YES && 0==H5T_cmp(src, dst)) {
*pcdata = &noop_cdata;
diff --git a/src/H5Z.c b/src/H5Z.c
index 993cc5e..5c7f0fa 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -268,7 +268,7 @@ H5Z_compress (const H5O_compress_t *comp, size_t nbytes, const void *src,
} else if (ret_value>=nbytes) {
/* Output is not smaller than input */
#ifdef H5Z_DEBUG
- H5Z_g[method].comp.over += 1;
+ H5Z_g[method].comp.over += nbytes;
over = 1;
#endif
HGOTO_DONE (0);
diff --git a/test/dsets.c b/test/dsets.c
index fab1f75..a341abe 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -36,6 +36,9 @@
#define DSET_SIMPLE_IO_NAME "simple_io"
#define DSET_TCONV_NAME "tconv"
#define DSET_COMPRESS_NAME "compressed"
+#define DSET_BOGUS_NAME "bogus"
+
+#define H5Z_BOGUS 255
/*-------------------------------------------------------------------------
@@ -353,6 +356,33 @@ test_tconv(hid_t file)
/*-------------------------------------------------------------------------
+ * Function: bogus
+ *
+ * Purpose: A bogus compression method.
+ *
+ * Return: Success: SRC_NBYTES, see compression documentation.
+ *
+ * Failure: 0
+ *
+ * Programmer: Robb Matzke
+ * Tuesday, April 21, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+bogus (unsigned int flags,
+ size_t cd_size, const void *client_data,
+ size_t src_nbytes, const void *src,
+ size_t dst_nbytes, void *dst/*out*/)
+{
+ memcpy (dst, src, src_nbytes);
+ return src_nbytes;
+}
+
+
+/*-------------------------------------------------------------------------
* Function: test_compression
*
* Purpose: Tests dataset compression.
@@ -374,48 +404,89 @@ test_compression(hid_t file)
hid_t dataset, space, xfer, dc;
herr_t status;
int points[100][200], check[100][200];
- int i, j, n;
- hsize_t dims[2], chunk_size[2];
+ const hsize_t size[2] = {100, 200};
+ const hsize_t chunk_size[2] = {2, 25};
+ const hssize_t hs_offset[2] = {7, 30};
+ const hsize_t hs_size[2] = {4, 50};
+
+ hsize_t i, j, n;
void *tconv_buf = NULL;
- printf("%-70s", "Testing compression");
- fflush (stdout);
+ printf ("%-70s", "Testing compression (setup)");
+ fflush (stderr);
- /* Initialize the dataset */
- for (i = n = 0; i < 100; i++) {
- for (j = 0; j < 100; j++) {
- points[i][j] = n++;
- }
- }
-
/* Create the data space */
- dims[0] = 100;
- dims[1] = 200;
- space = H5Screate_simple(2, dims, NULL);
+ space = H5Screate_simple(2, size, NULL);
assert(space>=0);
/* Create a small conversion buffer to test strip mining */
- tconv_buf = malloc (1000);
xfer = H5Pcreate (H5P_DATASET_XFER);
assert (xfer>=0);
+#if 0
+ tconv_buf = malloc (1000);
status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL);
assert (status>=0);
+#endif
/* Use chunked storage with compression */
dc = H5Pcreate (H5P_DATASET_CREATE);
- chunk_size[0] = 2;
- chunk_size[1] = 25;
H5Pset_chunk (dc, 2, chunk_size);
H5Pset_deflate (dc, 6);
/* Create the dataset */
dataset = H5Dcreate(file, DSET_COMPRESS_NAME, H5T_NATIVE_INT, space, dc);
assert(dataset >= 0);
+ puts (" PASSED");
+
+ /*----------------------------------------------------------------------
+ * STEP 1: Read uninitialized data. It should be zero.
+ *----------------------------------------------------------------------
+ */
+ printf ("%-70s", "Testing compression (uninitialized read)");
+ fflush (stdout);
+
+ status = H5Dread (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
+ xfer, check);
+ if (status<0) goto error;
+
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
+ if (0!=check[i][j]) {
+ puts("*FAILED*");
+ printf(" Read a non-zero value.\n");
+ printf(" At index %lu,%lu\n",
+ (unsigned long)i, (unsigned long)j);
+ goto error;
+ }
+ }
+ }
+ puts (" PASSED");
+
+ /*----------------------------------------------------------------------
+ * STEP 2: Test compression by setting up a chunked dataset and writing
+ * to it.
+ *----------------------------------------------------------------------
+ */
+ printf("%-70s", "Testing compression (write)");
+ fflush (stdout);
+
+ for (i=n=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
+ points[i][j] = n++;
+ }
+ }
- /* Write the data to the dataset */
status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
xfer, points);
if (status<0) goto error;
+ puts (" PASSED");
+
+ /*----------------------------------------------------------------------
+ * STEP 3: Try to read the data we just wrote.
+ *----------------------------------------------------------------------
+ */
+ printf ("%-70s", "Testing compression (read)");
+ fflush (stdout);
/* Read the dataset back */
status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
@@ -423,23 +494,31 @@ test_compression(hid_t file)
if (status<0) goto error;
/* Check that the values read are the same as the values written */
- for (i = 0; i < 100; i++) {
- for (j = 0; j < 200; j++) {
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
if (points[i][j] != check[i][j]) {
puts("*FAILED*");
printf(" Read different values than written.\n");
- printf(" At index %d,%d\n", i, j);
+ printf(" At index %lu,%lu\n",
+ (unsigned long)i, (unsigned long)j);
goto error;
}
}
}
-
- /*
- * Write some random data to the dataset, hopefully causing chunks to be
- * reallocated as they grow.
+ puts (" PASSED");
+
+ /*----------------------------------------------------------------------
+ * STEP 4: Write new data over the top of the old data. The new data is
+ * random thus not very compressible, and will cause the chunks to move
+ * around as they grow. We only change values for the left half of the
+ * dataset although we rewrite the whole thing.
+ *----------------------------------------------------------------------
*/
- for (i=0; i<100; i++) {
- for (j=0; j<100; j++) {
+ printf ("%-70s", "Testing compression (modify)");
+ fflush (stdout);
+
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]/2; j++) {
points[i][j] = rand ();
}
}
@@ -454,23 +533,129 @@ test_compression(hid_t file)
if (status<0) goto error;
/* Check that the values read are the same as the values written */
- for (i = 0; i < 100; i++) {
- for (j = 0; j < 200; j++) {
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
if (points[i][j] != check[i][j]) {
puts("*FAILED*");
printf(" Read different values than written.\n");
- printf(" At index %d,%d\n", i, j);
+ printf(" At index %lu,%lu\n",
+ (unsigned long)i, (unsigned long)j);
goto error;
}
}
}
+ puts (" PASSED");
+ /*----------------------------------------------------------------------
+ * STEP 5: Close the dataset and then open it and read it again. This
+ * insures that the compression message is picked up properly from the
+ * object header.
+ *----------------------------------------------------------------------
+ */
+ printf ("%-70s", "Testing compression (re-open)");
+ fflush (stdout);
+ H5Dclose (dataset);
+ dataset = H5Dopen (file, DSET_COMPRESS_NAME);
+ status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
+ xfer, check);
+ if (status<0) goto error;
+
+ /* Check that the values read are the same as the values written */
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
+ if (points[i][j] != check[i][j]) {
+ puts("*FAILED*");
+ printf(" Read different values than written.\n");
+ printf(" At index %lu,%lu\n",
+ (unsigned long)i, (unsigned long)j);
+ goto error;
+ }
+ }
+ }
+ puts (" PASSED");
+
+
+ /*----------------------------------------------------------------------
+ * STEP 6: Test partial I/O by writing to and then reading from a
+ * hyperslab of the dataset. The hyperslab does not line up on chunk
+ * boundaries (we know that case already works from above tests).
+ *----------------------------------------------------------------------
+ */
+ printf ("%-70s", "Testing compression (partial I/O)");
+ fflush (stderr);
+
+ for (i=0; i<hs_size[0]; i++) {
+ for (j=0; j<hs_size[1]; j++) {
+ points[hs_offset[0]+i][hs_offset[1]+j] = rand ();
+ }
+ }
+ H5Sset_hyperslab (space, hs_offset, hs_size, NULL);
+
+ status = H5Dwrite (dataset, H5T_NATIVE_INT, space, space, xfer, points);
+ if (status<0) goto error;
+ status = H5Dread (dataset, H5T_NATIVE_INT, space, space, xfer, check);
+ if (status<0) goto error;
+ /* Check that the values read are the same as the values written */
+ for (i=0; i<hs_size[0]; i++) {
+ for (j=0; j<hs_size[1]; j++) {
+ if (points[hs_offset[0]+i][hs_offset[1]+j] !=
+ check[hs_offset[0]+i][hs_offset[1]+j]) {
+ puts("*FAILED*");
+ printf(" Read different values than written.\n");
+ printf(" At index %lu,%lu\n",
+ (unsigned long)(hs_offset[0]+i),
+ (unsigned long)(hs_offset[1]+j));
+ goto error;
+ }
+ }
+ }
+ puts (" PASSED");
- H5Dclose(dataset);
+ /*----------------------------------------------------------------------
+ * STEP 7: Register an application-defined compression method and use it
+ * to write and then read the dataset.
+ *----------------------------------------------------------------------
+ */
+ printf ("%-70s", "Testing compression (app-defined method)");
+ fflush (stdout);
- puts(" PASSED");
+ H5Zregister (H5Z_BOGUS, "bogus", bogus, bogus);
+ H5Pset_compression (dc, H5Z_BOGUS, 0, 0, NULL);
+ H5Dclose (dataset);
+ H5Sclose (space);
+ space = H5Screate_simple (2, size, NULL);
+ dataset = H5Dcreate (file, DSET_BOGUS_NAME, H5T_NATIVE_INT, space, dc);
+ assert (dataset>=0);
+
+ status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
+ xfer, points);
+ if (status<0) goto error;
+ status = H5Dread (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
+ xfer, check);
+ if (status<0) goto error;
+
+ for (i=0; i<size[0]; i++) {
+ for (j=0; j<size[1]; j++) {
+ if (points[i][j] != check[i][j]) {
+ puts("*FAILED*");
+ printf(" Read different values than written.\n");
+ printf(" At index %lu,%lu\n",
+ (unsigned long)i, (unsigned long)j);
+ goto error;
+ }
+ }
+ }
+ puts (" PASSED");
+
+
+
+ /*----------------------------------------------------------------------
+ * Cleanup
+ *----------------------------------------------------------------------
+ */
+ H5Dclose(dataset);
return 0;
error: