From 91a34f543da20aee0de3ad2ec2cc58f86750bcf6 Mon Sep 17 00:00:00 2001 From: Robb Matzke Date: Tue, 21 Apr 1998 15:32:07 -0500 Subject: [svn-r355] Changes since 19980417 ---------------------- This checkin completes the compression stuff almost two weeks ahead of schedule. It should now all be working and documented. ./Makefile.in ./config/commence.in A distclean causes the distribution makefile to be copied to Makefile again. ./src/H5D.c Opening a compressed dataset now works properly. ./src/H5P.c ./src/H5Ppublic.h The H5Pset_compression() and H5Pget_compression() now work as documented. ./src/H5T.c Fixed a bug that causes the library to crash sometimes if debugging is turned on for the `t' package. ./src/H5Z.c Fixed a bug where the number of bytes attributed to compression overruns was too low in the debugging output. ./test/dsets.c More compression tests: reading of uninitialized data, opening an existing dataset and reading it, partial I/O which isn't aligned on chunk boundaries, and use of application-defined compression methods. ./MANIFEST Removed ./test/dspace.c. --- MANIFEST | 1 - Makefile.in | 1 + config/commence.in | 1 + src/H5D.c | 8 ++ src/H5P.c | 33 ++++++- src/H5Ppublic.h | 8 +- src/H5T.c | 4 + src/H5Z.c | 2 +- test/dsets.c | 249 ++++++++++++++++++++++++++++++++++++++++++++++------- 9 files changed, 267 insertions(+), 40 deletions(-) diff --git a/MANIFEST b/MANIFEST index b4cafd4..7875183 100644 --- a/MANIFEST +++ b/MANIFEST @@ -195,7 +195,6 @@ ./test/big.c ./test/cmpd_dset.c ./test/dsets.c -./test/dspace.c ./test/dtypes.c ./test/extend.c ./test/external.c diff --git a/Makefile.in b/Makefile.in index 9b7ef97..0ab8db6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -81,6 +81,7 @@ distclean: $(RM) config/commence config/conclude config/depend $(RM) config.cache config.log config.status src/H5config.h $(RM) Makefile + $(CP) Makefile.dist Makefile maintainer-clean: @echo "This command is intented for maintainers to use;" diff --git a/config/commence.in b/config/commence.in index c85da9b..7b41932 100644 --- a/config/commence.in +++ b/config/commence.in @@ -18,6 +18,7 @@ LIBS=@LIBS@ AR=@AR@ RANLIB=@RANLIB@ RM=rm -f +CP=cp INSTALL=@INSTALL@ INSTALL_PROGRAM=@INSTALL_PROGRAM@ INSTALL_DATA=@INSTALL_DATA@ diff --git a/src/H5D.c b/src/H5D.c index a49727a..45ed334 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -969,6 +969,14 @@ H5D_open(H5G_t *loc, const char *name) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to load type or space info from dataset header"); } + + /* Get the optional compression message */ + if (NULL==H5O_read (&(dataset->ent), H5O_COMPRESS, 0, + &(dataset->create_parms->compress))) { + H5E_clear (); + HDmemset (&(dataset->create_parms->compress), 0, + sizeof(dataset->create_parms->compress)); + } /* * Get the raw data layout info. It's actually stored in two locations: diff --git a/src/H5P.c b/src/H5P.c index b141853..3c802e4 100644 --- a/src/H5P.c +++ b/src/H5P.c @@ -1859,6 +1859,10 @@ H5Pget_preserve (hid_t plist_id) * specific compression initialization functions like * H5Pset_deflate(). * + * The FLAGS, CD_SIZE, and CLIENT_DATA are copied to the + * property list and eventually to the file and passed to the + * compression functions. + * * Return: Success: SUCCEED * * Failure: FAIL @@ -1871,7 +1875,8 @@ H5Pget_preserve (hid_t plist_id) *------------------------------------------------------------------------- */ herr_t -H5Pset_compression (hid_t plist_id, H5Z_method_t method) +H5Pset_compression (hid_t plist_id, H5Z_method_t method, unsigned int flags, + size_t cd_size, const void *client_data) { H5D_create_t *plist = NULL; @@ -1891,7 +1896,12 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method) /* Clear any previous compression method info, then set new value */ H5O_reset (H5O_COMPRESS, &(plist->compress)); plist->compress.method = method; - + plist->compress.flags = flags; + plist->compress.cd_size = cd_size; + if (cd_size) { + plist->compress.client_data = H5MM_xmalloc (cd_size); + HDmemcpy (plist->compress.client_data, client_data, cd_size); + } FUNC_LEAVE (SUCCEED); } @@ -1900,7 +1910,11 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method) * Function: H5Pget_compression * * Purpose: Gets the compression method information from a dataset - * creation property list. + * creation property list. The CLIENT_DATA buffer is initially + * CD_SIZE bytes. On return, CLIENT_DATA will be initialized + * with at most that many bytes, and CD_SIZE will contain the + * actual size of the client data, which might be larger than + * its original value. * * Return: Success: Compression method. * @@ -1914,7 +1928,8 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method) *------------------------------------------------------------------------- */ H5Z_method_t -H5Pget_compression (hid_t plist_id) +H5Pget_compression (hid_t plist_id, unsigned int *flags/*out*/, + size_t *cd_size/*in,out*/, void *client_data/*out*/) { H5D_create_t *plist = NULL; @@ -1927,6 +1942,16 @@ H5Pget_compression (hid_t plist_id) "not a dataset creation property list"); } + /* Output values */ + if (flags) *flags = plist->compress.flags; + if (cd_size) { + if (*cd_size>0 && client_data) { + HDmemcpy (client_data, plist->compress.client_data, + MIN(plist->compress.cd_size, *cd_size)); + } + *cd_size = plist->compress.cd_size; + } + FUNC_LEAVE (plist->compress.method); } diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bd87342..503f8c5 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -88,8 +88,12 @@ herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg); size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/); herr_t H5Pset_preserve (hid_t plist_id, hbool_t status); int H5Pget_preserve (hid_t plist_id); -herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method); -H5Z_method_t H5Pget_compression (hid_t plist_id); +herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method, + unsigned int flags, size_t cd_size, + const void *client_data); +H5Z_method_t H5Pget_compression (hid_t plist_id, unsigned int *flags/*out*/, + size_t *cd_size/*in,out*/, + void *client_data/*out*/); herr_t H5Pset_deflate (hid_t plist_id, int level); int H5Pget_deflate (hid_t plist_id); diff --git a/src/H5T.c b/src/H5T.c index 488e30d..0ca0377 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -3424,6 +3424,10 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg, FUNC_ENTER(H5T_find, NULL); + if (!noop_cdata.stats) { + noop_cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t)); + } + /* No-op case */ if (need_bkg=nbytes) { /* Output is not smaller than input */ #ifdef H5Z_DEBUG - H5Z_g[method].comp.over += 1; + H5Z_g[method].comp.over += nbytes; over = 1; #endif HGOTO_DONE (0); diff --git a/test/dsets.c b/test/dsets.c index fab1f75..a341abe 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -36,6 +36,9 @@ #define DSET_SIMPLE_IO_NAME "simple_io" #define DSET_TCONV_NAME "tconv" #define DSET_COMPRESS_NAME "compressed" +#define DSET_BOGUS_NAME "bogus" + +#define H5Z_BOGUS 255 /*------------------------------------------------------------------------- @@ -353,6 +356,33 @@ test_tconv(hid_t file) /*------------------------------------------------------------------------- + * Function: bogus + * + * Purpose: A bogus compression method. + * + * Return: Success: SRC_NBYTES, see compression documentation. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Tuesday, April 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +bogus (unsigned int flags, + size_t cd_size, const void *client_data, + size_t src_nbytes, const void *src, + size_t dst_nbytes, void *dst/*out*/) +{ + memcpy (dst, src, src_nbytes); + return src_nbytes; +} + + +/*------------------------------------------------------------------------- * Function: test_compression * * Purpose: Tests dataset compression. @@ -374,48 +404,89 @@ test_compression(hid_t file) hid_t dataset, space, xfer, dc; herr_t status; int points[100][200], check[100][200]; - int i, j, n; - hsize_t dims[2], chunk_size[2]; + const hsize_t size[2] = {100, 200}; + const hsize_t chunk_size[2] = {2, 25}; + const hssize_t hs_offset[2] = {7, 30}; + const hsize_t hs_size[2] = {4, 50}; + + hsize_t i, j, n; void *tconv_buf = NULL; - printf("%-70s", "Testing compression"); - fflush (stdout); + printf ("%-70s", "Testing compression (setup)"); + fflush (stderr); - /* Initialize the dataset */ - for (i = n = 0; i < 100; i++) { - for (j = 0; j < 100; j++) { - points[i][j] = n++; - } - } - /* Create the data space */ - dims[0] = 100; - dims[1] = 200; - space = H5Screate_simple(2, dims, NULL); + space = H5Screate_simple(2, size, NULL); assert(space>=0); /* Create a small conversion buffer to test strip mining */ - tconv_buf = malloc (1000); xfer = H5Pcreate (H5P_DATASET_XFER); assert (xfer>=0); +#if 0 + tconv_buf = malloc (1000); status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL); assert (status>=0); +#endif /* Use chunked storage with compression */ dc = H5Pcreate (H5P_DATASET_CREATE); - chunk_size[0] = 2; - chunk_size[1] = 25; H5Pset_chunk (dc, 2, chunk_size); H5Pset_deflate (dc, 6); /* Create the dataset */ dataset = H5Dcreate(file, DSET_COMPRESS_NAME, H5T_NATIVE_INT, space, dc); assert(dataset >= 0); + puts (" PASSED"); + + /*---------------------------------------------------------------------- + * STEP 1: Read uninitialized data. It should be zero. + *---------------------------------------------------------------------- + */ + printf ("%-70s", "Testing compression (uninitialized read)"); + fflush (stdout); + + status = H5Dread (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, check); + if (status<0) goto error; + + for (i=0; i=0); + + status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, points); + if (status<0) goto error; + status = H5Dread (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, check); + if (status<0) goto error; + + for (i=0; i