summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/H5.c105
-rw-r--r--src/H5AC.c10
-rw-r--r--src/H5B.c60
-rw-r--r--src/H5Bprivate.h1
-rw-r--r--src/H5D.c66
-rw-r--r--src/H5Distore.c665
-rw-r--r--src/H5Dprivate.h1
-rw-r--r--src/H5Dpublic.h39
-rw-r--r--src/H5E.c1
-rw-r--r--src/H5Epublic.h1
-rw-r--r--src/H5Farray.c33
-rw-r--r--src/H5Fistore.c665
-rw-r--r--src/H5Flow.c2
-rw-r--r--src/H5Fprivate.h7
-rw-r--r--src/H5Gnode.c1
-rw-r--r--src/H5MF.c86
-rw-r--r--src/H5MFprivate.h4
-rw-r--r--src/H5MM.c15
-rw-r--r--src/H5MMprivate.h2
-rw-r--r--src/H5O.c2
-rw-r--r--src/H5Ocomp.c291
-rw-r--r--src/H5Oprivate.h14
-rw-r--r--src/H5P.c173
-rw-r--r--src/H5Ppublic.h5
-rw-r--r--src/H5Sprivate.h8
-rw-r--r--src/H5Ssimp.c24
-rw-r--r--src/H5T.c133
-rw-r--r--src/H5Tconv.c24
-rw-r--r--src/H5Tpkg.h17
-rw-r--r--src/H5Tprivate.h2
-rw-r--r--src/H5Tpublic.h9
-rw-r--r--src/H5Z.c501
-rw-r--r--src/H5Zprivate.h23
-rw-r--r--src/H5Zpublic.h69
-rw-r--r--src/H5config.h.in9
-rw-r--r--src/H5private.h13
-rw-r--r--src/Makefile.in12
-rw-r--r--src/debug.c11
-rw-r--r--src/h5ls.c25
-rw-r--r--src/hdf5.h2
40 files changed, 2461 insertions, 670 deletions
diff --git a/src/H5.c b/src/H5.c
index 4d7896c..acdb5d1 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -38,6 +38,8 @@ static char RcsId[] = "@(#)$Revision$";
#include <ctype.h>
#include <errno.h>
#include <stdarg.h>
+#include <sys/time.h>
+#include <sys/resource.h>
/* private headers */
#include <H5private.h> /*library */
@@ -478,7 +480,7 @@ HDfprintf (FILE *stream, const char *fmt, ...)
if ('.'==*s) {
s++;
if (isdigit (*s)) {
- prec = (int)strtol (s+1, &rest, 10);
+ prec = (int)strtol (s, &rest, 10);
s = rest;
} else if ('*'==*s) {
prec = va_arg (ap, int);
@@ -576,6 +578,7 @@ HDfprintf (FILE *stream, const char *fmt, ...)
long double x = va_arg (ap, long double);
n = fprintf (stream, template, x);
}
+ break;
case 'a':
if (1) {
@@ -738,3 +741,103 @@ HDstrtoll (const char *s, const char **rest, int base)
if (rest) *rest = s;
return acc;
}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5_timer_reset
+ *
+ * Purpose: Resets the timer struct to zero. Use this to reset a timer
+ * that's being used as an accumulator for summing times.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5_timer_reset (H5_timer_t *timer)
+{
+ assert (timer);
+ HDmemset (timer, 0, sizeof *timer);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5_timer_begin
+ *
+ * Purpose: Initialize a timer to time something.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5_timer_begin (H5_timer_t *timer)
+{
+#ifdef HAVE_GETRUSAGE
+ struct rusage rusage;
+#endif
+ struct timeval etime;
+
+ assert (timer);
+
+#ifdef HAVE_GETRUSAGE
+ getrusage (RUSAGE_SELF, &rusage);
+ timer->utime = rusage.ru_utime.tv_sec + rusage.ru_utime.tv_usec/1e6;
+ timer->stime = rusage.ru_stime.tv_sec + rusage.ru_stime.tv_usec/1e6;
+#else
+ timer->utime = 0.0;
+ timer->stime = 0.0;
+#endif
+
+ gettimeofday (&etime, NULL);
+ timer->etime = etime.tv_sec + etime.tv_usec/1e6;
+}
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5_timer_end
+ *
+ * Purpose: This function should be called at the end of a timed region.
+ * The SUM is an optional pointer which will accumulate times.
+ * TMS is the same struct that was passed to H5_timer_start().
+ * On return, TMS will contain total times for the timed region.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/)
+{
+ H5_timer_t now;
+
+ assert (timer);
+ H5_timer_begin (&now);
+
+ timer->utime = now.utime - timer->utime;
+ timer->stime = now.stime - timer->stime;
+ timer->etime = now.etime - timer->etime;
+
+ if (sum) {
+ sum->utime += timer->utime;
+ sum->stime += timer->stime;
+ sum->etime += timer->etime;
+ }
+}
+
diff --git a/src/H5AC.c b/src/H5AC.c
index b666a97..98e7dfa 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -838,9 +838,11 @@ H5AC_debug(H5F_t *f)
FUNC_ENTER(H5AC_debug, FAIL);
- fprintf(stderr, "H5AC: cache diagnostics for %s\n", f->name);
- fprintf(stderr, " %18s %8s %8s %8s %8s+%-8s\n",
- "", "Hits", "Misses", "MissRate", "Inits", "Flushes");
+ fprintf(stderr, "H5AC: cache statistics for file %s\n", f->name);
+ fprintf(stderr, " %-18s %8s %8s %8s %8s+%-8s\n",
+ "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes");
+ fprintf(stderr, " %-18s %8s %8s %8s %8s-%-8s\n",
+ "-----", "----", "------", "--------", "-----", "-------");
for (i = H5AC_BT_ID; i < H5AC_NTYPES; i++) {
@@ -876,7 +878,7 @@ H5AC_debug(H5F_t *f)
} else {
sprintf(ascii, "%7.2f%%", miss_rate);
}
- fprintf(stderr, " %18s: %8u %8u %7s %8u%+-9ld\n", s,
+ fprintf(stderr, " %-18s %8u %8u %7s %8u%+-9ld\n", s,
cache->diagnostics[i].nhits,
cache->diagnostics[i].nmisses,
ascii,
diff --git a/src/H5B.c b/src/H5B.c
index 62bb783..69b64cb1 100644
--- a/src/H5B.c
+++ b/src/H5B.c
@@ -1022,7 +1022,7 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
uint8 *lt_key, hbool_t *lt_key_changed,
uint8 *md_key, void *udata,
uint8 *rt_key, hbool_t *rt_key_changed,
- haddr_t *new_node /*out */ )
+ haddr_t *new_node/*out*/)
{
H5B_t *bt = NULL, *twin = NULL, *tmp_bt = NULL;
intn lt = 0, idx = -1, rt, cmp = -1;
@@ -1097,10 +1097,9 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
idx = 0;
if (type->follow_min) {
- if ((my_ins = (type->insert) (f, bt->child + idx,
- bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
+ if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey,
+ lt_key_changed, md_key, udata,
+ bt->key[idx+1].nkey, rt_key_changed,
&child_addr/*out*/)) < 0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"unable to insert first leaf node");
@@ -1119,11 +1118,11 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR,
"unable to decode key");
}
- if ((my_ins = H5B_insert_helper(f, bt->child + idx, type,
+ if ((my_ins = H5B_insert_helper(f, bt->child+idx, type,
bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
- &child_addr /*out */ )) < 0) {
+ md_key, udata, bt->key[idx+1].nkey,
+ rt_key_changed,
+ &child_addr/*out*/))<0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"can't insert minimum subtree");
}
@@ -1138,11 +1137,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR,
"unable to decode key");
}
- if ((my_ins = (type->insert) (f, bt->child + idx,
- bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
- &child_addr /*out */ )) < 0) {
+ if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey,
+ lt_key_changed, md_key, udata,
+ bt->key[idx+1].nkey, rt_key_changed,
+ &child_addr/*out*/)) < 0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"can't insert minimum leaf node");
}
@@ -1176,11 +1174,11 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR,
"unable to decode key");
}
- if ((my_ins = H5B_insert_helper(f, bt->child + idx, type,
+ if ((my_ins = H5B_insert_helper(f, bt->child+idx, type,
bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
- &child_addr /*out */ )) < 0) {
+ md_key, udata, bt->key[idx+1].nkey,
+ rt_key_changed,
+ &child_addr/*out*/)) < 0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"can't insert maximum subtree");
}
@@ -1195,11 +1193,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR,
"unable to decode key");
}
- if ((my_ins = (type->insert) (f, bt->child + idx,
- bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
- &child_addr /*out */ )) < 0) {
+ if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey,
+ lt_key_changed, md_key, udata,
+ bt->key[idx+1].nkey, rt_key_changed,
+ &child_addr/*out*/)) < 0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"can't insert maximum leaf node");
}
@@ -1248,11 +1245,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type,
* Follow a branch out of this node to a leaf node of some other type.
*/
assert(idx >= 0 && idx < bt->nchildren);
- if ((my_ins = (type->insert) (f, bt->child + idx,
- bt->key[idx].nkey, lt_key_changed,
- md_key, udata,
- bt->key[idx + 1].nkey, rt_key_changed,
- &child_addr /*out */ )) < 0) {
+ if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey,
+ lt_key_changed, md_key, udata,
+ bt->key[idx+1].nkey, rt_key_changed,
+ &child_addr/*out*/)) < 0) {
HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR,
"can't insert leaf node");
}
@@ -1531,7 +1527,7 @@ H5B_nodesize(H5F_t *f, const H5B_class_t *type,
*-------------------------------------------------------------------------
*/
herr_t
-H5B_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
+H5B_debug(H5F_t *f, const haddr_t *addr, FILE *stream, intn indent,
intn fwidth, const H5B_class_t *type, void *udata)
{
H5B_t *bt = NULL;
@@ -1599,6 +1595,12 @@ H5B_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
"Address:");
H5F_addr_print(stream, bt->child + i);
fprintf(stream, "\n");
+
+ H5B_decode_key(f, bt, i);
+ if (type->debug_key) {
+ (type->debug_key)(stream, indent+3, MAX (0, fwidth-3),
+ bt->key[i].nkey, udata);
+ }
}
FUNC_LEAVE(SUCCEED);
diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h
index b108db8..40aa65c 100644
--- a/src/H5Bprivate.h
+++ b/src/H5Bprivate.h
@@ -86,6 +86,7 @@ typedef struct H5B_class_t {
herr_t (*list) (H5F_t*, const haddr_t*, void*); /*walk leaf nodes*/
herr_t (*decode) (H5F_t*, struct H5B_t*, uint8*, void*);
herr_t (*encode) (H5F_t*, struct H5B_t*, uint8*, void*);
+ herr_t (*debug_key)(FILE*, intn, intn, const void*, const void*);
} H5B_class_t;
/*
diff --git a/src/H5D.c b/src/H5D.c
index b53ba33..a49727a 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -17,16 +17,17 @@ static char RcsId[] = "@(#)$Revision$";
/* $Id$ */
#include <H5private.h> /* Generic Functions */
-#include <H5Iprivate.h> /* IDs */
+#include <H5Iprivate.h> /* IDs */
#include <H5ACprivate.h> /* Cache */
#include <H5Dprivate.h> /* Dataset functions */
#include <H5Eprivate.h> /* Error handling */
#include <H5Gprivate.h> /* Group headers */
-#include <H5HLprivate.h> /* Name heap */
+#include <H5HLprivate.h> /* Name heap */
#include <H5MFprivate.h> /* File space allocation header */
#include <H5MMprivate.h> /* Memory management */
#include <H5Oprivate.h> /* Object headers */
#include <H5Pprivate.h> /* Property lists */
+#include <H5Zprivate.h> /* Data compression */
#define PABLO_MASK H5D_mask
@@ -55,10 +56,17 @@ const H5D_create_t H5D_create_dflt = {
1, 1, 1, 1, 1, 1, 1, 1, /*...are quite useless. Larger chunks..*/
1, 1, 1, 1, 1, 1, 1, 1, /*...produce fewer, but larger I/O......*/
1, 1, 1, 1, 1, 1, 1, 1}, /*...requests. */
+
+ /* External file list */
{H5F_ADDR_UNDEF, /* External file list heap address */
0, /*...slots allocated */
0, /*...slots used */
- NULL} /*...slot array */
+ NULL}, /*...slot array */
+
+ /* Compression */
+ {H5Z_NONE, /* No compression */
+ 0, /*...flags */
+ 0, NULL} /*...client data */
};
/* Default dataset transfer property list */
@@ -741,6 +749,11 @@ H5D_create(H5G_t *loc, const char *name, const H5T_t *type, const H5S_t *space,
assert (type);
assert (space);
assert (create_parms);
+ if (H5Z_NONE!=create_parms->compress.method &&
+ H5D_CHUNKED!=create_parms->layout) {
+ HGOTO_ERROR (H5E_DATASET, H5E_BADVALUE, NULL,
+ "compression can only be used with chunked layout");
+ }
/* Initialize the dataset object */
new_dset = H5MM_xcalloc(1, sizeof(H5D_t));
@@ -829,9 +842,17 @@ H5D_create(H5G_t *loc, const char *name, const H5T_t *type, const H5S_t *space,
new_dset->type) < 0 ||
H5S_modify(&(new_dset->ent), new_dset->space) < 0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
- "can't update type or space header messages");
+ "unable to update type or space header messages");
}
+ /* Update the compression message */
+ if (H5Z_NONE!=new_dset->create_parms->compress.method &&
+ H5O_modify (&(new_dset->ent), H5O_COMPRESS, 0, H5O_FLAG_CONSTANT,
+ &(new_dset->create_parms->compress))<0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
+ "unable to update compression header message");
+ }
+
/*
* Initialize storage. We assume that external storage is already
* initialized by the caller, or at least will be before I/O is
@@ -1111,6 +1132,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
size_t target_size; /*desired buffer size */
size_t request_nelmts; /*requested strip mine */
H5T_bkg_t need_bkg; /*type of background buf*/
+#ifdef H5T_DEBUG
+ H5_timer_t timer;
+#endif
#ifdef HAVE_PARALLEL
int access_mode_saved = -1;
#endif
@@ -1210,6 +1234,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->read) {
status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->compress),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf/*out*/);
@@ -1296,6 +1321,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
* if necessary.
*/
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->compress),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, smine_start, smine_nelmts,
@@ -1314,14 +1340,19 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
/*
* Perform data type conversion.
*/
+#ifdef H5T_DEBUG
+ H5T_timer_begin (&timer, cdata);
+#endif
cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
- bkg_buf)<0) {
+ status = (tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf);
+#ifdef H5T_DEBUG
+ H5T_timer_end (&timer, cdata, smine_nelmts);
+#endif
+ if (status<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
- cdata->nelmts += smine_nelmts;
/*
* Scatter the data into memory.
@@ -1396,6 +1427,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
size_t target_size; /*desired buffer size */
size_t request_nelmts; /*requested strip mine */
H5T_bkg_t need_bkg; /*type of background buf*/
+#ifdef H5T_DEBUG
+ H5_timer_t timer;
+#endif
#ifdef HAVE_PARALLEL
int access_mode_saved = -1;
#endif
@@ -1495,6 +1529,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->write) {
status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->compress),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf);
@@ -1588,6 +1623,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
if ((H5D_OPTIMIZE_PIPE && H5T_BKG_YES==need_bkg) ||
(!H5D_OPTIMIZE_PIPE && need_bkg)) {
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->compress),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, smine_start, smine_nelmts,
@@ -1600,19 +1636,25 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
/*
* Perform data type conversion.
*/
+#ifdef H5T_DEBUG
+ H5T_timer_begin (&timer, cdata);
+#endif
cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
- bkg_buf)<0) {
+ status = (tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf);
+#ifdef H5T_DEBUG
+ H5T_timer_end (&timer, cdata, smine_nelmts);
+#endif
+ if (status<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
- cdata->nelmts += smine_nelmts;
/*
* Scatter the data out to the file.
*/
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->compress),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, smine_start, smine_nelmts,
diff --git a/src/H5Distore.c b/src/H5Distore.c
index 1f84aab..7e80f1a 100644
--- a/src/H5Distore.c
+++ b/src/H5Distore.c
@@ -14,17 +14,9 @@
#include <H5Oprivate.h>
#include <H5Vprivate.h>
-typedef enum H5F_isop_t {
- H5F_ISTORE_READ, /*read from file to memory */
- H5F_ISTORE_WRITE /*write from memory to file */
-} H5F_isop_t;
-
-/* Does the array domain include negative indices? */
-#undef H5F_ISTORE_NEGATIVE_DOMAIN
-
-#define PABLO_MASK H5F_istore_mask
/* Interface initialization */
+#define PABLO_MASK H5F_istore_mask
static hbool_t interface_initialize_g = FALSE;
#define INTERFACE_INIT NULL
@@ -48,13 +40,8 @@ static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8 *raw,
void *_key);
static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8 *raw,
void *_key);
-static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout,
- H5F_isop_t op,
- const hssize_t offset_f[],
- const hsize_t size[],
- const hssize_t offset_m[],
- const hsize_t size_m[],
- void *buf);
+static herr_t H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth,
+ const void *key, const void *udata);
/*
* B-tree key. A key contains the minimum logical N-dimensional address and
@@ -68,12 +55,11 @@ static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout,
* Only the first few values of the OFFSET and SIZE fields are actually
* stored on disk, depending on the dimensionality.
*
- * The storage file address is part of the B-tree and not part of the key.
+ * The chunk's file address is part of the B-tree and not part of the key.
*/
typedef struct H5F_istore_key_t {
- uintn file_number; /*external file number */
+ hsize_t nbytes; /*size of stored data */
hssize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
- hsize_t size[H5O_LAYOUT_NDIMS]; /*logical chunk size */
} H5F_istore_key_t;
typedef struct H5F_istore_ud1_t {
@@ -97,6 +83,7 @@ H5B_class_t H5B_ISTORE[1] = {{
NULL, /*list */
H5F_istore_decode_key, /*decode */
H5F_istore_encode_key, /*encode */
+ H5F_istore_debug_key, /*debug */
}};
@@ -128,9 +115,8 @@ H5F_istore_sizeof_rkey(H5F_t __unused__ *f, const void *_udata)
assert(udata);
assert(udata->mesg.ndims > 0 && udata->mesg.ndims <= H5O_LAYOUT_NDIMS);
- nbytes = 4 + /*external file number */
- udata->mesg.ndims * 4 + /*dimension indices */
- udata->mesg.ndims * 4; /*dimension sizes */
+ nbytes = 4 + /*storage size */
+ udata->mesg.ndims * 4; /*dimension indices */
return nbytes;
}
@@ -157,7 +143,7 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
{
H5F_istore_key_t *key = (H5F_istore_key_t *) _key;
intn i;
- intn ndims = (intn)(bt->sizeof_rkey/8);
+ intn ndims = (intn)((bt->sizeof_rkey-4)/4);
FUNC_ENTER(H5F_istore_decode_key, FAIL);
@@ -169,11 +155,9 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS);
/* decode */
- UINT32DECODE(raw, key->file_number);
- assert(0 == key->file_number);
+ UINT32DECODE (raw, key->nbytes);
for (i = 0; i < ndims; i++) {
UINT32DECODE(raw, key->offset[i]);
- UINT32DECODE(raw, key->size[i]);
}
FUNC_LEAVE(SUCCEED);
@@ -200,7 +184,7 @@ static herr_t
H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
{
H5F_istore_key_t *key = (H5F_istore_key_t *) _key;
- intn ndims = (intn)(bt->sizeof_rkey / 8);
+ intn ndims = (intn)((bt->sizeof_rkey-4) / 4);
intn i;
FUNC_ENTER(H5F_istore_encode_key, FAIL);
@@ -213,11 +197,9 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS);
/* encode */
- UINT32ENCODE(raw, key->file_number);
- assert(0 == key->file_number);
+ UINT32ENCODE (raw, key->nbytes);
for (i = 0; i < ndims; i++) {
UINT32ENCODE(raw, key->offset[i]);
- UINT32ENCODE(raw, key->size[i]);
}
FUNC_LEAVE(SUCCEED);
@@ -225,6 +207,46 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
/*-------------------------------------------------------------------------
+ * Function: H5F_istore_debug_key
+ *
+ * Purpose: Prints a key.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth,
+ const void *_key, const void *_udata)
+{
+ const H5F_istore_key_t *key = (const H5F_istore_key_t *)_key;
+ const H5F_istore_ud1_t *udata = (const H5F_istore_ud1_t *)_udata;
+ int i;
+
+ FUNC_ENTER (H5F_istore_debug_key, FAIL);
+ assert (key);
+
+ HDfprintf (stream, "%*s%-*s %Hd bytes\n", indent, "", fwidth,
+ "Chunk size:", key->nbytes);
+ HDfprintf (stream, "%*s%-*s {", indent, "", fwidth,
+ "Logical offset:");
+ for (i=0; i<udata->mesg.ndims; i++) {
+ HDfprintf (stream, "%s%Hd", i?", ":"", key->offset[i]);
+ }
+ fputs ("}\n", stream);
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5F_istore_cmp2
*
* Purpose: Compares two keys sort of like strcmp(). The UDATA pointer
@@ -352,7 +374,6 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op,
H5F_istore_key_t *lt_key = (H5F_istore_key_t *) _lt_key;
H5F_istore_key_t *rt_key = (H5F_istore_key_t *) _rt_key;
H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata;
- hsize_t nbytes;
intn i;
FUNC_ENTER(H5F_istore_new_node, FAIL);
@@ -366,37 +387,34 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op,
assert(addr);
/* Allocate new storage */
- nbytes = H5V_vector_reduce_product(udata->mesg.ndims, udata->key.size);
- assert(nbytes > 0);
- if (H5MF_alloc(f, H5MF_RAW, nbytes, addr /*out */ ) < 0) {
+ assert (udata->key.nbytes > 0);
+ if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, addr /*out */ ) < 0) {
HRETURN_ERROR(H5E_IO, H5E_CANTINIT, FAIL,
"couldn't allocate new file storage");
}
udata->addr = *addr;
- udata->key.file_number = 0;
- lt_key->file_number = udata->key.file_number;
- if (H5B_INS_LEFT != op)
- rt_key->file_number = 0;
- /* Initialize the key(s) */
- for (i = 0; i < udata->mesg.ndims; i++) {
- /*
- * The left key describes the storage of the UDATA chunk being
- * inserted into the tree.
- */
- assert(udata->key.size[i] > 0);
+ /*
+ * The left key describes the storage of the UDATA chunk being
+ * inserted into the tree.
+ */
+ lt_key->nbytes = udata->key.nbytes;
+ for (i=0; i<udata->mesg.ndims; i++) {
lt_key->offset[i] = udata->key.offset[i];
- lt_key->size[i] = udata->key.size[i];
+ }
- /*
- * The right key might already be present. If not, then add
- * a zero-width chunk.
- */
- if (H5B_INS_LEFT != op) {
- assert (udata->key.size[i] < MAX_HSSIZET);
+ /*
+ * The right key might already be present. If not, then add a zero-width
+ * chunk.
+ */
+ if (H5B_INS_LEFT != op) {
+ rt_key->nbytes = 0;
+ for (i=0; i<udata->mesg.ndims; i++) {
+ assert (udata->mesg.dim[i] < MAX_HSSIZET);
+ assert (udata->key.offset[i]+(hssize_t)(udata->mesg.dim[i]) >
+ udata->key.offset[i]);
rt_key->offset[i] = udata->key.offset[i] +
- (hssize_t)udata->key.size[i];
- rt_key->size[i] = 0;
+ (hssize_t)(udata->mesg.dim[i]);
}
}
@@ -445,12 +463,10 @@ H5F_istore_found(H5F_t __unused__ *f, const haddr_t *addr,
/* Initialize return values */
udata->addr = *addr;
- udata->key.file_number = lt_key->file_number;
- assert(0 == lt_key->file_number);
+ udata->key.nbytes = lt_key->nbytes;
+ assert (lt_key->nbytes>0);
for (i = 0; i < udata->mesg.ndims; i++) {
udata->key.offset[i] = lt_key->offset[i];
- udata->key.size[i] = lt_key->size[i];
- assert(lt_key->size[i] > 0);
}
FUNC_LEAVE(SUCCEED);
@@ -500,7 +516,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata;
intn i, cmp;
H5B_ins_t ret_value = H5B_INS_ERROR;
- hsize_t nbytes;
FUNC_ENTER(H5F_istore_insert, H5B_INS_ERROR);
@@ -523,46 +538,61 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
assert("HDF5 INTERNAL ERROR -- see rpm" && 0);
HRETURN_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, H5B_INS_ERROR,
"internal error");
-
- } else if (H5V_hyper_eq(udata->mesg.ndims,
- udata->key.offset, udata->key.size,
- lt_key->offset, lt_key->size)) {
+
+ } else if (H5V_vector_eq_s (udata->mesg.ndims,
+ udata->key.offset, lt_key->offset) &&
+ lt_key->nbytes>0) {
/*
- * Already exists. Just return the info.
+ * Already exists. If the new size is not the same as the old size
+ * then we should reallocate storage.
*/
+#if 1
+ if (lt_key->nbytes != udata->key.nbytes) {
+ if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr,
+ udata->key.nbytes, new_node/*out*/)<0) {
+ HRETURN_ERROR (H5E_STORAGE, H5E_WRITEERROR, H5B_INS_ERROR,
+ "unable to reallocate chunk storage");
+ }
+ lt_key->nbytes = udata->key.nbytes;
+ *lt_key_changed = TRUE;
+ udata->addr = *new_node;
+ ret_value = H5B_INS_CHANGE;
+ } else {
+ udata->addr = *addr;
+ ret_value = H5B_INS_NOOP;
+ }
+#else
+ assert (lt_key->nbytes == udata->key.nbytes);
+ assert (!H5F_addr_defined (&(udata->addr)) ||
+ H5F_addr_eq (&(udata->addr), addr));
udata->addr = *addr;
- udata->key.file_number = lt_key->file_number;
ret_value = H5B_INS_NOOP;
+#endif
} else if (H5V_hyper_disjointp(udata->mesg.ndims,
- lt_key->offset, lt_key->size,
- udata->key.offset, udata->key.size)) {
+ lt_key->offset, udata->mesg.dim,
+ udata->key.offset, udata->mesg.dim)) {
assert(H5V_hyper_disjointp(udata->mesg.ndims,
- rt_key->offset, rt_key->size,
- udata->key.offset, udata->key.size));
-
+ rt_key->offset, udata->mesg.dim,
+ udata->key.offset, udata->mesg.dim));
/*
* Split this node, inserting the new new node to the right of the
* current node. The MD_KEY is where the split occurs.
*/
- md_key->file_number = udata->key.file_number;
- for (i=0, nbytes=1; i<udata->mesg.ndims; i++) {
+ md_key->nbytes = udata->key.nbytes;
+ for (i=0; i<udata->mesg.ndims; i++) {
assert(0 == udata->key.offset[i] % udata->mesg.dim[i]);
- assert(udata->key.size[i] == udata->mesg.dim[i]);
md_key->offset[i] = udata->key.offset[i];
- md_key->size[i] = udata->key.size[i];
- nbytes *= udata->key.size[i];
}
/*
* Allocate storage for the new chunk
*/
- if (H5MF_alloc(f, H5MF_RAW, nbytes, new_node /*out */ ) < 0) {
+ if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, new_node/*out*/)<0) {
HRETURN_ERROR(H5E_IO, H5E_CANTINIT, H5B_INS_ERROR,
"file allocation failed");
}
udata->addr = *new_node;
- udata->key.file_number = 0;
ret_value = H5B_INS_RIGHT;
} else {
@@ -576,43 +606,29 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
/*-------------------------------------------------------------------------
- * Function: H5F_istore_copy_hyperslab
- *
- * Purpose: Reads or writes a hyperslab to disk depending on whether OP
- * is H5F_ISTORE_READ or H5F_ISTORE_WRITE. The hyperslab
- * storage is described with ISTORE and exists in file F. The
- * file hyperslab begins at location OFFSET_F[] (an N-dimensional
- * point in the domain in terms of elements) in the file and
- * OFFSET_M[] in memory pointed to by BUF. Its size is SIZE[]
- * elements. The dimensionality of memory is assumed to be the
- * same as the file and the total size of the multi-dimensional
- * memory buffer is SIZE_M[].
- *
- * The slowest varying dimension is always listed first in the
- * various offset and size arrays.
- *
- * A `chunk' is a hyperslab of the disk array which is stored
- * contiguously. I/O occurs in units of chunks where the size of
- * a chunk is determined by the alignment constraints specified
- * in ISTORE.
+ * Function: H5F_istore_read
+ *
+ * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw
+ * storage array.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
- * Friday, October 17, 1997
+ * Wednesday, October 15, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
-static herr_t
-H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
- const hssize_t offset_f[], const hsize_t size[],
- const hssize_t offset_m[], const hsize_t size_m[],
- void *buf/*in or out*/)
+herr_t
+H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
+ const H5O_compress_t *comp, const hssize_t offset_f[],
+ const hsize_t size[], void *buf)
{
+ hssize_t offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t size_m[H5O_LAYOUT_NDIMS];
intn i, carry;
hsize_t idx_cur[H5O_LAYOUT_NDIMS];
hsize_t idx_min[H5O_LAYOUT_NDIMS];
@@ -621,161 +637,156 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS];
hssize_t sub_offset_m[H5O_LAYOUT_NDIMS];
size_t chunk_size;
- hsize_t acc;
- uint8 *chunk = NULL;
+ uint8 *chunk=NULL, *compressed=NULL;
H5F_istore_ud1_t udata;
herr_t status;
herr_t ret_value = FAIL;
- FUNC_ENTER(H5F_istore_copy_hyperslab, FAIL);
+ FUNC_ENTER(H5F_istore_read, FAIL);
- /* check args */
- assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(H5F_addr_defined(&(layout->addr)));
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
- assert(H5F_ISTORE_READ == op || H5F_ISTORE_WRITE == op);
- assert(size);
- assert(size_m);
- assert(buf);
+ /* Check args */
+ assert (f);
+ assert (layout && H5D_CHUNKED==layout->type);
+ assert (layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert (H5F_addr_defined(&(layout->addr)));
+ assert (offset_f);
+ assert (size);
+ assert (buf);
+
+ /*
+ * For now, a hyperslab of the file must be read into an array in
+ * memory.We do not yet support reading into a hyperslab of memory.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ offset_m[i] = 0;
+ size_m[i] = size[i];
+ }
+
#ifndef NDEBUG
for (i=0; i<layout->ndims; i++) {
- assert(size_m[i] > 0); /*destination must exist */
- /*hyperslab must fit in BUF */
- assert((offset_m?offset_m[i]:0) + size[i] <= size_m[i]);
- assert(layout->dim[i] > 0);
+ assert (offset_f[i]>=0); /*negative offsets not supported*/
+ assert (offset_m[i]>=0); /*negative offsets not supported*/
+ assert (size[i]<MAX_SIZET);
+ assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]);
+ assert(layout->dim[i]>0);
}
#endif
+ /* Determine the chunk size and allocate buffers */
+ for (i=0, chunk_size=1; i<layout->ndims; i++) {
+ chunk_size *= layout->dim[i];
+ }
+ chunk = H5MM_xmalloc(chunk_size);
+ if (comp && H5Z_NONE!=comp->method) {
+ compressed = H5MM_xmalloc (chunk_size);
+ }
+
/*
- * As a special case of H5F_ISTORE_READ, if the source is aligned on
- * a chunk boundary and is the same size as a chunk, and the destination
- * is the same size as a chunk, then instead of reading into a temporary
- * buffer and then into the destination, we read directly into the
- * destination.
+ * As a special case if the source is aligned on a chunk boundary and is
+ * the same size as a chunk, and the destination is the same size as a
+ * chunk, then instead of reading into a temporary buffer and then into
+ * the destination, we read directly into the destination.
*/
- if (H5F_ISTORE_READ==op) {
- for (i=0, acc=1; i<layout->ndims; i++) {
- if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/
- if (size[i]!=layout->dim[i]) break; /*src not a chunk*/
- if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/
- udata.key.offset[i] = offset_f[i];
- udata.key.size[i] = layout->dim[i];
- acc *= layout->dim[i];
- }
- chunk_size = acc;
- assert ((hsize_t)chunk_size==acc);
-
- if (i==layout->ndims) {
- udata.mesg = *layout;
- H5F_addr_undef (&(udata.addr));
- udata.key.file_number = 0;
- status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata);
- if (status>=0 && H5F_addr_defined (&(udata.addr))) {
- assert (0==udata.key.file_number);
- if (H5F_block_read (f, &(udata.addr), (hsize_t)chunk_size,
- buf)<0) {
+ for (i=0; i<layout->ndims; i++) {
+ if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/
+ if (size[i]!=layout->dim[i]) break; /*src not a chunk*/
+ if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/
+ udata.key.offset[i] = offset_f[i];
+ }
+ if (i==layout->ndims) {
+ udata.mesg = *layout;
+ H5F_addr_undef (&(udata.addr));
+ status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata);
+ if (status>=0 && H5F_addr_defined (&(udata.addr))) {
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read (f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
"unable to read raw storage chunk");
}
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size, buf)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress raw storage chunk");
+ }
} else {
- HDmemset (buf, 0, chunk_size);
+ assert (udata.key.nbytes==chunk_size);
+ if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
}
- HRETURN (SUCCEED);
+ } else {
+ HDmemset (buf, 0, chunk_size);
}
+ HGOTO_DONE (SUCCEED);
}
/*
* This is the general case. We set up multi-dimensional counters
- * (idx_min, idx_max, and idx_cur) and loop through the chunks copying
- * each chunk into a temporary buffer, compressing or decompressing, and
- * then copying it to it's destination.
+ * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing
+ * or copying each chunk into a temporary buffer, and then copying it to
+ * it's destination.
*/
for (i=0; i<layout->ndims; i++) {
- idx_min[i] = (offset_f?offset_f[i]:0) / layout->dim[i];
- idx_max[i] = ((offset_f?offset_f[i]:0) + size[i]-1) / layout->dim[i]+1;
+ idx_min[i] = offset_f[i] / layout->dim[i];
+ idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1;
idx_cur[i] = idx_min[i];
}
- /* Allocate buffers */
- for (i=0, chunk_size=1; i<layout->ndims; i++) {
- chunk_size *= layout->dim[i];
- }
- chunk = H5MM_xmalloc(chunk_size);
-
/* Initialize non-changing part of udata */
udata.mesg = *layout;
/* Loop over all chunks */
while (1) {
- /* Read/Write chunk or create it if it doesn't exist */
- udata.mesg.ndims = layout->ndims;
- H5F_addr_undef(&(udata.addr));
- udata.key.file_number = 0;
-
for (i=0; i<layout->ndims; i++) {
-
/* The location and size of the chunk being accessed */
assert (layout->dim[i] < MAX_HSSIZET);
udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]);
- udata.key.size[i] = layout->dim[i];
/* The offset and size wrt the chunk */
- offset_wrt_chunk[i] = MAX((offset_f?offset_f[i]:0),
- udata.key.offset[i]) -
+ offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) -
udata.key.offset[i];
- sub_size[i] = MIN((idx_cur[i] + 1) * layout->dim[i],
- (offset_f ? offset_f[i] : 0) + size[i]) -
+ sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i],
+ offset_f[i]+size[i]) -
(udata.key.offset[i] + offset_wrt_chunk[i]);
/* Offset into mem buffer */
sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] +
- (offset_m ? offset_m[i] : 0) -
- (offset_f ? offset_f[i] : 0);
- }
-
- if (H5F_ISTORE_WRITE == op) {
- status = H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata);
- assert(status >= 0);
- } else {
- status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata);
+ offset_m[i] - offset_f[i];
}
- /*
- * If the operation is reading from the disk or if we are writing a
- * partial chunk then load the chunk from disk.
- */
- if (H5F_ISTORE_READ == op ||
- !H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) ||
- !H5V_vector_eq_u(layout->ndims, sub_size, udata.key.size)) {
- if (status>=0 && H5F_addr_defined(&(udata.addr))) {
- assert(0==udata.key.file_number);
- if (H5F_block_read(f, &(udata.addr), (hsize_t)chunk_size,
- chunk) < 0) {
+ /* Read chunk */
+ H5F_addr_undef(&(udata.addr));
+ status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata);
+ if (status>=0 && H5F_addr_defined(&(udata.addr))) {
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read (f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size,
+ chunk)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress data");
+ }
+ } else {
+ assert (udata.key.nbytes == chunk_size);
+ if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) {
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
"unable to read raw storage chunk");
}
- } else {
- HDmemset(chunk, 0, chunk_size);
- }
- }
- /* Transfer data to/from the chunk */
- if (H5F_ISTORE_WRITE==op) {
- H5V_hyper_copy(layout->ndims, sub_size,
- udata.key.size, offset_wrt_chunk, chunk,
- size_m, sub_offset_m, buf);
- assert(0 == udata.key.file_number);
- if (H5F_block_write(f, &(udata.addr), (hsize_t)chunk_size,
- chunk) < 0) {
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "unable to write raw storage chunk");
}
} else {
- H5V_hyper_copy(layout->ndims, sub_size,
- size_m, sub_offset_m, (void *)buf,
- udata.key.size, offset_wrt_chunk, chunk);
+ HDmemset(chunk, 0, chunk_size);
}
+
+ /* Transfer data from the chunk buffer to the application */
+ H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m,
+ (void *)buf, layout->dim, offset_wrt_chunk, chunk);
/* Increment indices */
for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) {
@@ -787,51 +798,13 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
ret_value = SUCCEED;
done:
- chunk = H5MM_xfree(chunk);
+ H5MM_xfree(chunk);
+ H5MM_xfree (compressed);
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
- * Function: H5F_istore_read
- *
- * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw
- * storage array.
- *
- * Return: Success: SUCCEED
- *
- * Failure: FAIL
- *
- * Programmer: Robb Matzke
- * Wednesday, October 15, 1997
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
- const hssize_t offset[], const hsize_t size[], void *buf)
-{
- FUNC_ENTER(H5F_istore_read, FAIL);
-
- /* Check args */
- assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
- assert(size);
- assert(buf);
-
- if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_READ,
- offset, size, H5V_ZERO, size, buf) < 0) {
- HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
- "hyperslab output failure");
- }
- FUNC_LEAVE(SUCCEED);
-}
-
-
-/*-------------------------------------------------------------------------
* Function: H5F_istore_write
*
* Purpose: Writes a multi-dimensional buffer to (part of) an indexed raw
@@ -850,25 +823,174 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
*/
herr_t
H5F_istore_write(H5F_t *f, const H5O_layout_t *layout,
- const hssize_t offset[], const hsize_t size[],
- const void *buf)
+ const H5O_compress_t *comp, const hssize_t offset_f[],
+ const hsize_t size[], const void *buf)
{
+ hssize_t offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t size_m[H5O_LAYOUT_NDIMS];
+ intn i, carry;
+ hsize_t idx_cur[H5O_LAYOUT_NDIMS];
+ hsize_t idx_min[H5O_LAYOUT_NDIMS];
+ hsize_t idx_max[H5O_LAYOUT_NDIMS];
+ hsize_t sub_size[H5O_LAYOUT_NDIMS];
+ hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS];
+ hssize_t sub_offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t chunk_size, nbytes;
+ uint8 *chunk=NULL, *compressed=NULL, *outbuf;
+ H5F_istore_ud1_t udata;
+ herr_t ret_value = FAIL;
+
FUNC_ENTER(H5F_istore_write, FAIL);
/* Check args */
assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(&(layout->addr)));
+ assert (offset_f);
assert(size);
assert(buf);
- if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_WRITE,
- offset, size, H5V_ZERO, size,
- (void*)buf) < 0) {
- HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "hyperslab output failure");
+ /*
+ * For now the source must not be a hyperslab. It must be an entire
+ * memory buffer.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ offset_m[i] = 0;
+ size_m[i] = size[i];
+ }
+
+#ifndef NDEBUG
+ for (i=0; i<layout->ndims; i++) {
+ assert (offset_f[i]>=0); /*negative offsets not supported*/
+ assert (offset_m[i]>=0); /*negative offsets not supported*/
+ assert(size[i]<MAX_SIZET);
+ assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]);
+ assert(layout->dim[i]>0);
}
- FUNC_LEAVE(SUCCEED);
+#endif
+
+ /*
+ * This is the general case. We set up multi-dimensional counters
+ * (idx_min, idx_max, and idx_cur) and loop through the chunks copying
+ * each chunk into a temporary buffer, compressing or decompressing, and
+ * then copying it to it's destination.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ idx_min[i] = offset_f[i] / layout->dim[i];
+ idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1;
+ idx_cur[i] = idx_min[i];
+ }
+
+ /* Allocate buffers */
+ for (i=0, chunk_size=1; i<layout->ndims; i++) {
+ chunk_size *= layout->dim[i];
+ }
+ chunk = H5MM_xmalloc(chunk_size);
+ if (comp && H5Z_NONE!=comp->method) {
+ compressed = H5MM_xmalloc (chunk_size);
+ }
+
+ /* Initialize non-changing part of udata */
+ udata.mesg = *layout;
+
+ /* Loop over all chunks */
+ while (1) {
+
+ for (i=0; i<layout->ndims; i++) {
+ /* The location and size of the chunk being accessed */
+ assert (layout->dim[i] < MAX_HSSIZET);
+ udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]);
+
+ /* The offset and size wrt the chunk */
+ offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) -
+ udata.key.offset[i];
+ sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i],
+ offset_f[i]+size[i]) -
+ (udata.key.offset[i] + offset_wrt_chunk[i]);
+
+ /* Offset into mem buffer */
+ sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] +
+ offset_m[i] - offset_f[i];
+ }
+
+ /*
+ * If we are writing a partial chunk then load the chunk from disk
+ * and uncompress it if it exists.
+ */
+ if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) ||
+ !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) {
+ if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 &&
+ H5F_addr_defined (&(udata.addr))) {
+
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read(f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size,
+ chunk)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress data");
+ }
+ } else {
+ assert (chunk_size==udata.key.nbytes);
+ if (H5F_block_read(f, &(udata.addr), udata.key.nbytes,
+ chunk)<0) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ }
+ } else {
+ HDmemset(chunk, 0, chunk_size);
+ }
+ }
+
+ /* Transfer data to the chunk */
+ H5V_hyper_copy(layout->ndims, sub_size,
+ layout->dim, offset_wrt_chunk, chunk,
+ size_m, sub_offset_m, buf);
+
+ /* Compress the chunk */
+ if (compressed &&
+ (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) &&
+ nbytes<chunk_size) {
+ outbuf = compressed;
+ } else {
+ outbuf = chunk;
+ nbytes = chunk_size;
+ }
+
+ /*
+ * Create the chunk it if it doesn't exist, or reallocate the chunk
+ * if its size changed. Then write the data into the file.
+ */
+ H5F_addr_undef(&(udata.addr));
+ udata.key.nbytes = nbytes;
+ if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+ "unable to allocate chunk");
+ }
+ if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) {
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "unable to write raw storage chunk");
+ }
+
+ /* Increment indices */
+ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) {
+ if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i];
+ else carry = 0;
+ }
+ if (carry) break;
+ }
+ ret_value = SUCCEED;
+
+ done:
+ H5MM_xfree(chunk);
+ H5MM_xfree (compressed);
+ FUNC_LEAVE(ret_value);
}
@@ -921,3 +1043,36 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ )
FUNC_LEAVE(SUCCEED);
}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_debug
+ *
+ * Purpose: Debugs a B-tree node for indexed raw data storage.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
+ intn fwidth, int ndims)
+{
+ H5F_istore_ud1_t udata;
+
+ FUNC_ENTER (H5F_istore_debug, FAIL);
+
+ HDmemset (&udata, 0, sizeof udata);
+ udata.mesg.ndims = ndims;
+
+ H5B_debug (f, addr, stream, indent, fwidth, H5B_ISTORE, &udata);
+
+ FUNC_LEAVE (SUCCEED);
+}
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index f39e5de..e437ce7 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -46,6 +46,7 @@ typedef struct H5D_create_t {
intn chunk_ndims; /*chunk dimensionality */
hsize_t chunk_size[32]; /*chunk size if chunked storage */
H5O_efl_t efl; /*external file list */
+ H5O_compress_t compress; /*data compression parameters */
} H5D_create_t;
/* Dataset transfer property list */
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index 634b5b9..d1d5c26 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -1,13 +1,13 @@
/****************************************************************************
- * NCSA HDF *
- * Software Development Group *
- * National Center for Supercomputing Applications *
- * University of Illinois at Urbana-Champaign *
- * 605 E. Springfield, Champaign IL 61820 *
- * *
- * For conditions of distribution and use, see the accompanying *
- * hdf/COPYING file. *
- * *
+ * NCSA HDF *
+ * Software Development Group *
+ * National Center for Supercomputing Applications *
+ * University of Illinois at Urbana-Champaign *
+ * 605 E. Springfield, Champaign IL 61820 *
+ * *
+ * For conditions of distribution and use, see the accompanying *
+ * hdf/COPYING file. *
+ * *
****************************************************************************/
/*
@@ -22,38 +22,39 @@
/* Values for the H5D_LAYOUT property */
typedef enum H5D_layout_t {
- H5D_LAYOUT_ERROR = -1,
+ H5D_LAYOUT_ERROR = -1,
- H5D_COMPACT = 0, /*raw data is very small */
- H5D_CONTIGUOUS = 1, /*the default */
- H5D_CHUNKED = 2, /*slow and fancy */
+ H5D_COMPACT = 0, /*raw data is very small */
+ H5D_CONTIGUOUS = 1, /*the default */
+ H5D_CHUNKED = 2, /*slow and fancy */
- H5D_NLAYOUTS = 3 /*this one must be last! */
+ H5D_NLAYOUTS = 3 /*this one must be last! */
} H5D_layout_t;
#ifdef HAVE_PARALLEL
/* Values for the data transfer property */
typedef enum H5D_transfer_t {
- H5D_XFER_INDEPENDENT, /*Independent data transfer */
- H5D_XFER_COLLECTIVE /*Collective data transfer */
+ H5D_XFER_INDEPENDENT, /*Independent data transfer */
+ H5D_XFER_COLLECTIVE /*Collective data transfer */
} H5D_transfer_t;
#endif
+
#ifdef __cplusplus
extern "C" {
#endif
hid_t H5Dcreate (hid_t file_id, const char *name, hid_t type_id,
- hid_t space_id, hid_t create_parms_id);
+ hid_t space_id, hid_t create_parms_id);
hid_t H5Dopen (hid_t file_id, const char *name);
herr_t H5Dclose (hid_t dataset_id);
hid_t H5Dget_space (hid_t dataset_id);
hid_t H5Dget_type (hid_t dataset_id);
hid_t H5Dget_create_plist (hid_t dataset_id);
herr_t H5Dread (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
- hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/);
+ hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/);
herr_t H5Dwrite (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
- hid_t file_space_id, hid_t xfer_parms_id, const void *buf);
+ hid_t file_space_id, hid_t xfer_parms_id, const void *buf);
herr_t H5Dextend (hid_t dataset_id, const hsize_t *size);
#ifdef __cplusplus
diff --git a/src/H5E.c b/src/H5E.c
index 02a3852..5062d77 100644
--- a/src/H5E.c
+++ b/src/H5E.c
@@ -54,6 +54,7 @@ static const H5E_major_mesg_t H5E_major_mesg_g[] = {
{H5E_DATASET, "Dataset interface"},
{H5E_STORAGE, "Data storage layer"},
{H5E_TEMPLATE, "Property list interface"},
+ {H5E_COMP, "Data compression layer"},
{H5E_EFL, "External file list"},
};
diff --git a/src/H5Epublic.h b/src/H5Epublic.h
index ee228e2..284b310 100644
--- a/src/H5Epublic.h
+++ b/src/H5Epublic.h
@@ -46,6 +46,7 @@ typedef enum H5E_major_t {
H5E_DATASET, /*Dataset */
H5E_STORAGE, /*data storage */
H5E_TEMPLATE, /*Property lists */
+ H5E_COMP, /*Data compression */
H5E_EFL /*External file list */
} H5E_major_t;
diff --git a/src/H5Farray.c b/src/H5Farray.c
index 98d3022..1f2bb6c 100644
--- a/src/H5Farray.c
+++ b/src/H5Farray.c
@@ -112,7 +112,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/)
*/
herr_t
H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl,
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
const hsize_t _hslab_size[], const hsize_t mem_size[],
const hssize_t mem_offset[], const hssize_t file_offset[],
void *_buf/*out*/)
@@ -173,6 +173,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
"negative offsets are not valid");
}
}
+
+ /*
+ * Compression cannot be used for contiguous data.
+ */
+ if (comp && H5Z_NONE!=comp->method) {
+ HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "compression is not allowed for contiguous data");
+ }
/*
* Calculate the strides needed to walk through the array on disk
@@ -270,7 +278,8 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max);
"unable to copy into a proper hyperslab");
}
}
- if (H5F_istore_read (f, layout, file_offset, hslab_size, buf)<0) {
+ if (H5F_istore_read (f, layout, comp, file_offset, hslab_size,
+ buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, "chunked read failed");
}
break;
@@ -313,9 +322,10 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max);
*/
herr_t
H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl, const hsize_t _hslab_size[],
- const hsize_t mem_size[], const hssize_t mem_offset[],
- const hssize_t file_offset[], const void *_buf)
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
+ const hsize_t _hslab_size[], const hsize_t mem_size[],
+ const hssize_t mem_offset[], const hssize_t file_offset[],
+ const void *_buf)
{
const uint8 *buf = (const uint8 *)_buf; /*cast for arithmetic */
hssize_t file_stride[H5O_LAYOUT_NDIMS]; /*strides through file */
@@ -369,10 +379,18 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
*/
for (i=0; i<ndims; i++) {
if (mem_offset[i]<0 || file_offset[i]<0) {
- HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"negative offsets are not valid");
}
}
+
+ /*
+ * Compression cannot be used for contiguous data
+ */
+ if (comp && H5Z_NONE!=comp->method) {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+ "compression is not allowed for contiguous data");
+ }
/*
* Calculate the strides needed to walk through the array on disk.
@@ -471,7 +489,8 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max);
"unable to copy from a proper hyperslab");
}
}
- if (H5F_istore_write (f, layout, file_offset, hslab_size, buf)<0) {
+ if (H5F_istore_write (f, layout, comp, file_offset, hslab_size,
+ buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"chunked write failed");
}
diff --git a/src/H5Fistore.c b/src/H5Fistore.c
index 1f84aab..7e80f1a 100644
--- a/src/H5Fistore.c
+++ b/src/H5Fistore.c
@@ -14,17 +14,9 @@
#include <H5Oprivate.h>
#include <H5Vprivate.h>
-typedef enum H5F_isop_t {
- H5F_ISTORE_READ, /*read from file to memory */
- H5F_ISTORE_WRITE /*write from memory to file */
-} H5F_isop_t;
-
-/* Does the array domain include negative indices? */
-#undef H5F_ISTORE_NEGATIVE_DOMAIN
-
-#define PABLO_MASK H5F_istore_mask
/* Interface initialization */
+#define PABLO_MASK H5F_istore_mask
static hbool_t interface_initialize_g = FALSE;
#define INTERFACE_INIT NULL
@@ -48,13 +40,8 @@ static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8 *raw,
void *_key);
static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8 *raw,
void *_key);
-static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout,
- H5F_isop_t op,
- const hssize_t offset_f[],
- const hsize_t size[],
- const hssize_t offset_m[],
- const hsize_t size_m[],
- void *buf);
+static herr_t H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth,
+ const void *key, const void *udata);
/*
* B-tree key. A key contains the minimum logical N-dimensional address and
@@ -68,12 +55,11 @@ static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout,
* Only the first few values of the OFFSET and SIZE fields are actually
* stored on disk, depending on the dimensionality.
*
- * The storage file address is part of the B-tree and not part of the key.
+ * The chunk's file address is part of the B-tree and not part of the key.
*/
typedef struct H5F_istore_key_t {
- uintn file_number; /*external file number */
+ hsize_t nbytes; /*size of stored data */
hssize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
- hsize_t size[H5O_LAYOUT_NDIMS]; /*logical chunk size */
} H5F_istore_key_t;
typedef struct H5F_istore_ud1_t {
@@ -97,6 +83,7 @@ H5B_class_t H5B_ISTORE[1] = {{
NULL, /*list */
H5F_istore_decode_key, /*decode */
H5F_istore_encode_key, /*encode */
+ H5F_istore_debug_key, /*debug */
}};
@@ -128,9 +115,8 @@ H5F_istore_sizeof_rkey(H5F_t __unused__ *f, const void *_udata)
assert(udata);
assert(udata->mesg.ndims > 0 && udata->mesg.ndims <= H5O_LAYOUT_NDIMS);
- nbytes = 4 + /*external file number */
- udata->mesg.ndims * 4 + /*dimension indices */
- udata->mesg.ndims * 4; /*dimension sizes */
+ nbytes = 4 + /*storage size */
+ udata->mesg.ndims * 4; /*dimension indices */
return nbytes;
}
@@ -157,7 +143,7 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
{
H5F_istore_key_t *key = (H5F_istore_key_t *) _key;
intn i;
- intn ndims = (intn)(bt->sizeof_rkey/8);
+ intn ndims = (intn)((bt->sizeof_rkey-4)/4);
FUNC_ENTER(H5F_istore_decode_key, FAIL);
@@ -169,11 +155,9 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS);
/* decode */
- UINT32DECODE(raw, key->file_number);
- assert(0 == key->file_number);
+ UINT32DECODE (raw, key->nbytes);
for (i = 0; i < ndims; i++) {
UINT32DECODE(raw, key->offset[i]);
- UINT32DECODE(raw, key->size[i]);
}
FUNC_LEAVE(SUCCEED);
@@ -200,7 +184,7 @@ static herr_t
H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
{
H5F_istore_key_t *key = (H5F_istore_key_t *) _key;
- intn ndims = (intn)(bt->sizeof_rkey / 8);
+ intn ndims = (intn)((bt->sizeof_rkey-4) / 4);
intn i;
FUNC_ENTER(H5F_istore_encode_key, FAIL);
@@ -213,11 +197,9 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS);
/* encode */
- UINT32ENCODE(raw, key->file_number);
- assert(0 == key->file_number);
+ UINT32ENCODE (raw, key->nbytes);
for (i = 0; i < ndims; i++) {
UINT32ENCODE(raw, key->offset[i]);
- UINT32ENCODE(raw, key->size[i]);
}
FUNC_LEAVE(SUCCEED);
@@ -225,6 +207,46 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key)
/*-------------------------------------------------------------------------
+ * Function: H5F_istore_debug_key
+ *
+ * Purpose: Prints a key.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth,
+ const void *_key, const void *_udata)
+{
+ const H5F_istore_key_t *key = (const H5F_istore_key_t *)_key;
+ const H5F_istore_ud1_t *udata = (const H5F_istore_ud1_t *)_udata;
+ int i;
+
+ FUNC_ENTER (H5F_istore_debug_key, FAIL);
+ assert (key);
+
+ HDfprintf (stream, "%*s%-*s %Hd bytes\n", indent, "", fwidth,
+ "Chunk size:", key->nbytes);
+ HDfprintf (stream, "%*s%-*s {", indent, "", fwidth,
+ "Logical offset:");
+ for (i=0; i<udata->mesg.ndims; i++) {
+ HDfprintf (stream, "%s%Hd", i?", ":"", key->offset[i]);
+ }
+ fputs ("}\n", stream);
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5F_istore_cmp2
*
* Purpose: Compares two keys sort of like strcmp(). The UDATA pointer
@@ -352,7 +374,6 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op,
H5F_istore_key_t *lt_key = (H5F_istore_key_t *) _lt_key;
H5F_istore_key_t *rt_key = (H5F_istore_key_t *) _rt_key;
H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata;
- hsize_t nbytes;
intn i;
FUNC_ENTER(H5F_istore_new_node, FAIL);
@@ -366,37 +387,34 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op,
assert(addr);
/* Allocate new storage */
- nbytes = H5V_vector_reduce_product(udata->mesg.ndims, udata->key.size);
- assert(nbytes > 0);
- if (H5MF_alloc(f, H5MF_RAW, nbytes, addr /*out */ ) < 0) {
+ assert (udata->key.nbytes > 0);
+ if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, addr /*out */ ) < 0) {
HRETURN_ERROR(H5E_IO, H5E_CANTINIT, FAIL,
"couldn't allocate new file storage");
}
udata->addr = *addr;
- udata->key.file_number = 0;
- lt_key->file_number = udata->key.file_number;
- if (H5B_INS_LEFT != op)
- rt_key->file_number = 0;
- /* Initialize the key(s) */
- for (i = 0; i < udata->mesg.ndims; i++) {
- /*
- * The left key describes the storage of the UDATA chunk being
- * inserted into the tree.
- */
- assert(udata->key.size[i] > 0);
+ /*
+ * The left key describes the storage of the UDATA chunk being
+ * inserted into the tree.
+ */
+ lt_key->nbytes = udata->key.nbytes;
+ for (i=0; i<udata->mesg.ndims; i++) {
lt_key->offset[i] = udata->key.offset[i];
- lt_key->size[i] = udata->key.size[i];
+ }
- /*
- * The right key might already be present. If not, then add
- * a zero-width chunk.
- */
- if (H5B_INS_LEFT != op) {
- assert (udata->key.size[i] < MAX_HSSIZET);
+ /*
+ * The right key might already be present. If not, then add a zero-width
+ * chunk.
+ */
+ if (H5B_INS_LEFT != op) {
+ rt_key->nbytes = 0;
+ for (i=0; i<udata->mesg.ndims; i++) {
+ assert (udata->mesg.dim[i] < MAX_HSSIZET);
+ assert (udata->key.offset[i]+(hssize_t)(udata->mesg.dim[i]) >
+ udata->key.offset[i]);
rt_key->offset[i] = udata->key.offset[i] +
- (hssize_t)udata->key.size[i];
- rt_key->size[i] = 0;
+ (hssize_t)(udata->mesg.dim[i]);
}
}
@@ -445,12 +463,10 @@ H5F_istore_found(H5F_t __unused__ *f, const haddr_t *addr,
/* Initialize return values */
udata->addr = *addr;
- udata->key.file_number = lt_key->file_number;
- assert(0 == lt_key->file_number);
+ udata->key.nbytes = lt_key->nbytes;
+ assert (lt_key->nbytes>0);
for (i = 0; i < udata->mesg.ndims; i++) {
udata->key.offset[i] = lt_key->offset[i];
- udata->key.size[i] = lt_key->size[i];
- assert(lt_key->size[i] > 0);
}
FUNC_LEAVE(SUCCEED);
@@ -500,7 +516,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata;
intn i, cmp;
H5B_ins_t ret_value = H5B_INS_ERROR;
- hsize_t nbytes;
FUNC_ENTER(H5F_istore_insert, H5B_INS_ERROR);
@@ -523,46 +538,61 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
assert("HDF5 INTERNAL ERROR -- see rpm" && 0);
HRETURN_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, H5B_INS_ERROR,
"internal error");
-
- } else if (H5V_hyper_eq(udata->mesg.ndims,
- udata->key.offset, udata->key.size,
- lt_key->offset, lt_key->size)) {
+
+ } else if (H5V_vector_eq_s (udata->mesg.ndims,
+ udata->key.offset, lt_key->offset) &&
+ lt_key->nbytes>0) {
/*
- * Already exists. Just return the info.
+ * Already exists. If the new size is not the same as the old size
+ * then we should reallocate storage.
*/
+#if 1
+ if (lt_key->nbytes != udata->key.nbytes) {
+ if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr,
+ udata->key.nbytes, new_node/*out*/)<0) {
+ HRETURN_ERROR (H5E_STORAGE, H5E_WRITEERROR, H5B_INS_ERROR,
+ "unable to reallocate chunk storage");
+ }
+ lt_key->nbytes = udata->key.nbytes;
+ *lt_key_changed = TRUE;
+ udata->addr = *new_node;
+ ret_value = H5B_INS_CHANGE;
+ } else {
+ udata->addr = *addr;
+ ret_value = H5B_INS_NOOP;
+ }
+#else
+ assert (lt_key->nbytes == udata->key.nbytes);
+ assert (!H5F_addr_defined (&(udata->addr)) ||
+ H5F_addr_eq (&(udata->addr), addr));
udata->addr = *addr;
- udata->key.file_number = lt_key->file_number;
ret_value = H5B_INS_NOOP;
+#endif
} else if (H5V_hyper_disjointp(udata->mesg.ndims,
- lt_key->offset, lt_key->size,
- udata->key.offset, udata->key.size)) {
+ lt_key->offset, udata->mesg.dim,
+ udata->key.offset, udata->mesg.dim)) {
assert(H5V_hyper_disjointp(udata->mesg.ndims,
- rt_key->offset, rt_key->size,
- udata->key.offset, udata->key.size));
-
+ rt_key->offset, udata->mesg.dim,
+ udata->key.offset, udata->mesg.dim));
/*
* Split this node, inserting the new new node to the right of the
* current node. The MD_KEY is where the split occurs.
*/
- md_key->file_number = udata->key.file_number;
- for (i=0, nbytes=1; i<udata->mesg.ndims; i++) {
+ md_key->nbytes = udata->key.nbytes;
+ for (i=0; i<udata->mesg.ndims; i++) {
assert(0 == udata->key.offset[i] % udata->mesg.dim[i]);
- assert(udata->key.size[i] == udata->mesg.dim[i]);
md_key->offset[i] = udata->key.offset[i];
- md_key->size[i] = udata->key.size[i];
- nbytes *= udata->key.size[i];
}
/*
* Allocate storage for the new chunk
*/
- if (H5MF_alloc(f, H5MF_RAW, nbytes, new_node /*out */ ) < 0) {
+ if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, new_node/*out*/)<0) {
HRETURN_ERROR(H5E_IO, H5E_CANTINIT, H5B_INS_ERROR,
"file allocation failed");
}
udata->addr = *new_node;
- udata->key.file_number = 0;
ret_value = H5B_INS_RIGHT;
} else {
@@ -576,43 +606,29 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key,
/*-------------------------------------------------------------------------
- * Function: H5F_istore_copy_hyperslab
- *
- * Purpose: Reads or writes a hyperslab to disk depending on whether OP
- * is H5F_ISTORE_READ or H5F_ISTORE_WRITE. The hyperslab
- * storage is described with ISTORE and exists in file F. The
- * file hyperslab begins at location OFFSET_F[] (an N-dimensional
- * point in the domain in terms of elements) in the file and
- * OFFSET_M[] in memory pointed to by BUF. Its size is SIZE[]
- * elements. The dimensionality of memory is assumed to be the
- * same as the file and the total size of the multi-dimensional
- * memory buffer is SIZE_M[].
- *
- * The slowest varying dimension is always listed first in the
- * various offset and size arrays.
- *
- * A `chunk' is a hyperslab of the disk array which is stored
- * contiguously. I/O occurs in units of chunks where the size of
- * a chunk is determined by the alignment constraints specified
- * in ISTORE.
+ * Function: H5F_istore_read
+ *
+ * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw
+ * storage array.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
- * Friday, October 17, 1997
+ * Wednesday, October 15, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
-static herr_t
-H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
- const hssize_t offset_f[], const hsize_t size[],
- const hssize_t offset_m[], const hsize_t size_m[],
- void *buf/*in or out*/)
+herr_t
+H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
+ const H5O_compress_t *comp, const hssize_t offset_f[],
+ const hsize_t size[], void *buf)
{
+ hssize_t offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t size_m[H5O_LAYOUT_NDIMS];
intn i, carry;
hsize_t idx_cur[H5O_LAYOUT_NDIMS];
hsize_t idx_min[H5O_LAYOUT_NDIMS];
@@ -621,161 +637,156 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS];
hssize_t sub_offset_m[H5O_LAYOUT_NDIMS];
size_t chunk_size;
- hsize_t acc;
- uint8 *chunk = NULL;
+ uint8 *chunk=NULL, *compressed=NULL;
H5F_istore_ud1_t udata;
herr_t status;
herr_t ret_value = FAIL;
- FUNC_ENTER(H5F_istore_copy_hyperslab, FAIL);
+ FUNC_ENTER(H5F_istore_read, FAIL);
- /* check args */
- assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(H5F_addr_defined(&(layout->addr)));
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
- assert(H5F_ISTORE_READ == op || H5F_ISTORE_WRITE == op);
- assert(size);
- assert(size_m);
- assert(buf);
+ /* Check args */
+ assert (f);
+ assert (layout && H5D_CHUNKED==layout->type);
+ assert (layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert (H5F_addr_defined(&(layout->addr)));
+ assert (offset_f);
+ assert (size);
+ assert (buf);
+
+ /*
+ * For now, a hyperslab of the file must be read into an array in
+ * memory.We do not yet support reading into a hyperslab of memory.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ offset_m[i] = 0;
+ size_m[i] = size[i];
+ }
+
#ifndef NDEBUG
for (i=0; i<layout->ndims; i++) {
- assert(size_m[i] > 0); /*destination must exist */
- /*hyperslab must fit in BUF */
- assert((offset_m?offset_m[i]:0) + size[i] <= size_m[i]);
- assert(layout->dim[i] > 0);
+ assert (offset_f[i]>=0); /*negative offsets not supported*/
+ assert (offset_m[i]>=0); /*negative offsets not supported*/
+ assert (size[i]<MAX_SIZET);
+ assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]);
+ assert(layout->dim[i]>0);
}
#endif
+ /* Determine the chunk size and allocate buffers */
+ for (i=0, chunk_size=1; i<layout->ndims; i++) {
+ chunk_size *= layout->dim[i];
+ }
+ chunk = H5MM_xmalloc(chunk_size);
+ if (comp && H5Z_NONE!=comp->method) {
+ compressed = H5MM_xmalloc (chunk_size);
+ }
+
/*
- * As a special case of H5F_ISTORE_READ, if the source is aligned on
- * a chunk boundary and is the same size as a chunk, and the destination
- * is the same size as a chunk, then instead of reading into a temporary
- * buffer and then into the destination, we read directly into the
- * destination.
+ * As a special case if the source is aligned on a chunk boundary and is
+ * the same size as a chunk, and the destination is the same size as a
+ * chunk, then instead of reading into a temporary buffer and then into
+ * the destination, we read directly into the destination.
*/
- if (H5F_ISTORE_READ==op) {
- for (i=0, acc=1; i<layout->ndims; i++) {
- if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/
- if (size[i]!=layout->dim[i]) break; /*src not a chunk*/
- if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/
- udata.key.offset[i] = offset_f[i];
- udata.key.size[i] = layout->dim[i];
- acc *= layout->dim[i];
- }
- chunk_size = acc;
- assert ((hsize_t)chunk_size==acc);
-
- if (i==layout->ndims) {
- udata.mesg = *layout;
- H5F_addr_undef (&(udata.addr));
- udata.key.file_number = 0;
- status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata);
- if (status>=0 && H5F_addr_defined (&(udata.addr))) {
- assert (0==udata.key.file_number);
- if (H5F_block_read (f, &(udata.addr), (hsize_t)chunk_size,
- buf)<0) {
+ for (i=0; i<layout->ndims; i++) {
+ if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/
+ if (size[i]!=layout->dim[i]) break; /*src not a chunk*/
+ if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/
+ udata.key.offset[i] = offset_f[i];
+ }
+ if (i==layout->ndims) {
+ udata.mesg = *layout;
+ H5F_addr_undef (&(udata.addr));
+ status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata);
+ if (status>=0 && H5F_addr_defined (&(udata.addr))) {
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read (f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
"unable to read raw storage chunk");
}
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size, buf)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress raw storage chunk");
+ }
} else {
- HDmemset (buf, 0, chunk_size);
+ assert (udata.key.nbytes==chunk_size);
+ if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
}
- HRETURN (SUCCEED);
+ } else {
+ HDmemset (buf, 0, chunk_size);
}
+ HGOTO_DONE (SUCCEED);
}
/*
* This is the general case. We set up multi-dimensional counters
- * (idx_min, idx_max, and idx_cur) and loop through the chunks copying
- * each chunk into a temporary buffer, compressing or decompressing, and
- * then copying it to it's destination.
+ * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing
+ * or copying each chunk into a temporary buffer, and then copying it to
+ * it's destination.
*/
for (i=0; i<layout->ndims; i++) {
- idx_min[i] = (offset_f?offset_f[i]:0) / layout->dim[i];
- idx_max[i] = ((offset_f?offset_f[i]:0) + size[i]-1) / layout->dim[i]+1;
+ idx_min[i] = offset_f[i] / layout->dim[i];
+ idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1;
idx_cur[i] = idx_min[i];
}
- /* Allocate buffers */
- for (i=0, chunk_size=1; i<layout->ndims; i++) {
- chunk_size *= layout->dim[i];
- }
- chunk = H5MM_xmalloc(chunk_size);
-
/* Initialize non-changing part of udata */
udata.mesg = *layout;
/* Loop over all chunks */
while (1) {
- /* Read/Write chunk or create it if it doesn't exist */
- udata.mesg.ndims = layout->ndims;
- H5F_addr_undef(&(udata.addr));
- udata.key.file_number = 0;
-
for (i=0; i<layout->ndims; i++) {
-
/* The location and size of the chunk being accessed */
assert (layout->dim[i] < MAX_HSSIZET);
udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]);
- udata.key.size[i] = layout->dim[i];
/* The offset and size wrt the chunk */
- offset_wrt_chunk[i] = MAX((offset_f?offset_f[i]:0),
- udata.key.offset[i]) -
+ offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) -
udata.key.offset[i];
- sub_size[i] = MIN((idx_cur[i] + 1) * layout->dim[i],
- (offset_f ? offset_f[i] : 0) + size[i]) -
+ sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i],
+ offset_f[i]+size[i]) -
(udata.key.offset[i] + offset_wrt_chunk[i]);
/* Offset into mem buffer */
sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] +
- (offset_m ? offset_m[i] : 0) -
- (offset_f ? offset_f[i] : 0);
- }
-
- if (H5F_ISTORE_WRITE == op) {
- status = H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata);
- assert(status >= 0);
- } else {
- status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata);
+ offset_m[i] - offset_f[i];
}
- /*
- * If the operation is reading from the disk or if we are writing a
- * partial chunk then load the chunk from disk.
- */
- if (H5F_ISTORE_READ == op ||
- !H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) ||
- !H5V_vector_eq_u(layout->ndims, sub_size, udata.key.size)) {
- if (status>=0 && H5F_addr_defined(&(udata.addr))) {
- assert(0==udata.key.file_number);
- if (H5F_block_read(f, &(udata.addr), (hsize_t)chunk_size,
- chunk) < 0) {
+ /* Read chunk */
+ H5F_addr_undef(&(udata.addr));
+ status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata);
+ if (status>=0 && H5F_addr_defined(&(udata.addr))) {
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read (f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size,
+ chunk)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress data");
+ }
+ } else {
+ assert (udata.key.nbytes == chunk_size);
+ if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) {
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
"unable to read raw storage chunk");
}
- } else {
- HDmemset(chunk, 0, chunk_size);
- }
- }
- /* Transfer data to/from the chunk */
- if (H5F_ISTORE_WRITE==op) {
- H5V_hyper_copy(layout->ndims, sub_size,
- udata.key.size, offset_wrt_chunk, chunk,
- size_m, sub_offset_m, buf);
- assert(0 == udata.key.file_number);
- if (H5F_block_write(f, &(udata.addr), (hsize_t)chunk_size,
- chunk) < 0) {
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "unable to write raw storage chunk");
}
} else {
- H5V_hyper_copy(layout->ndims, sub_size,
- size_m, sub_offset_m, (void *)buf,
- udata.key.size, offset_wrt_chunk, chunk);
+ HDmemset(chunk, 0, chunk_size);
}
+
+ /* Transfer data from the chunk buffer to the application */
+ H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m,
+ (void *)buf, layout->dim, offset_wrt_chunk, chunk);
/* Increment indices */
for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) {
@@ -787,51 +798,13 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op,
ret_value = SUCCEED;
done:
- chunk = H5MM_xfree(chunk);
+ H5MM_xfree(chunk);
+ H5MM_xfree (compressed);
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
- * Function: H5F_istore_read
- *
- * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw
- * storage array.
- *
- * Return: Success: SUCCEED
- *
- * Failure: FAIL
- *
- * Programmer: Robb Matzke
- * Wednesday, October 15, 1997
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
- const hssize_t offset[], const hsize_t size[], void *buf)
-{
- FUNC_ENTER(H5F_istore_read, FAIL);
-
- /* Check args */
- assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
- assert(size);
- assert(buf);
-
- if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_READ,
- offset, size, H5V_ZERO, size, buf) < 0) {
- HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
- "hyperslab output failure");
- }
- FUNC_LEAVE(SUCCEED);
-}
-
-
-/*-------------------------------------------------------------------------
* Function: H5F_istore_write
*
* Purpose: Writes a multi-dimensional buffer to (part of) an indexed raw
@@ -850,25 +823,174 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout,
*/
herr_t
H5F_istore_write(H5F_t *f, const H5O_layout_t *layout,
- const hssize_t offset[], const hsize_t size[],
- const void *buf)
+ const H5O_compress_t *comp, const hssize_t offset_f[],
+ const hsize_t size[], const void *buf)
{
+ hssize_t offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t size_m[H5O_LAYOUT_NDIMS];
+ intn i, carry;
+ hsize_t idx_cur[H5O_LAYOUT_NDIMS];
+ hsize_t idx_min[H5O_LAYOUT_NDIMS];
+ hsize_t idx_max[H5O_LAYOUT_NDIMS];
+ hsize_t sub_size[H5O_LAYOUT_NDIMS];
+ hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS];
+ hssize_t sub_offset_m[H5O_LAYOUT_NDIMS];
+ hsize_t chunk_size, nbytes;
+ uint8 *chunk=NULL, *compressed=NULL, *outbuf;
+ H5F_istore_ud1_t udata;
+ herr_t ret_value = FAIL;
+
FUNC_ENTER(H5F_istore_write, FAIL);
/* Check args */
assert(f);
- assert(layout && H5D_CHUNKED == layout->type);
- assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(&(layout->addr)));
+ assert (offset_f);
assert(size);
assert(buf);
- if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_WRITE,
- offset, size, H5V_ZERO, size,
- (void*)buf) < 0) {
- HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "hyperslab output failure");
+ /*
+ * For now the source must not be a hyperslab. It must be an entire
+ * memory buffer.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ offset_m[i] = 0;
+ size_m[i] = size[i];
+ }
+
+#ifndef NDEBUG
+ for (i=0; i<layout->ndims; i++) {
+ assert (offset_f[i]>=0); /*negative offsets not supported*/
+ assert (offset_m[i]>=0); /*negative offsets not supported*/
+ assert(size[i]<MAX_SIZET);
+ assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]);
+ assert(layout->dim[i]>0);
}
- FUNC_LEAVE(SUCCEED);
+#endif
+
+ /*
+ * This is the general case. We set up multi-dimensional counters
+ * (idx_min, idx_max, and idx_cur) and loop through the chunks copying
+ * each chunk into a temporary buffer, compressing or decompressing, and
+ * then copying it to it's destination.
+ */
+ for (i=0; i<layout->ndims; i++) {
+ idx_min[i] = offset_f[i] / layout->dim[i];
+ idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1;
+ idx_cur[i] = idx_min[i];
+ }
+
+ /* Allocate buffers */
+ for (i=0, chunk_size=1; i<layout->ndims; i++) {
+ chunk_size *= layout->dim[i];
+ }
+ chunk = H5MM_xmalloc(chunk_size);
+ if (comp && H5Z_NONE!=comp->method) {
+ compressed = H5MM_xmalloc (chunk_size);
+ }
+
+ /* Initialize non-changing part of udata */
+ udata.mesg = *layout;
+
+ /* Loop over all chunks */
+ while (1) {
+
+ for (i=0; i<layout->ndims; i++) {
+ /* The location and size of the chunk being accessed */
+ assert (layout->dim[i] < MAX_HSSIZET);
+ udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]);
+
+ /* The offset and size wrt the chunk */
+ offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) -
+ udata.key.offset[i];
+ sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i],
+ offset_f[i]+size[i]) -
+ (udata.key.offset[i] + offset_wrt_chunk[i]);
+
+ /* Offset into mem buffer */
+ sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] +
+ offset_m[i] - offset_f[i];
+ }
+
+ /*
+ * If we are writing a partial chunk then load the chunk from disk
+ * and uncompress it if it exists.
+ */
+ if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) ||
+ !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) {
+ if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 &&
+ H5F_addr_defined (&(udata.addr))) {
+
+ if (compressed && udata.key.nbytes<chunk_size) {
+ if (H5F_block_read(f, &(udata.addr), udata.key.nbytes,
+ compressed)<0) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes,
+ compressed, chunk_size,
+ chunk)) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "unable to uncompress data");
+ }
+ } else {
+ assert (chunk_size==udata.key.nbytes);
+ if (H5F_block_read(f, &(udata.addr), udata.key.nbytes,
+ chunk)<0) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "unable to read raw storage chunk");
+ }
+ }
+ } else {
+ HDmemset(chunk, 0, chunk_size);
+ }
+ }
+
+ /* Transfer data to the chunk */
+ H5V_hyper_copy(layout->ndims, sub_size,
+ layout->dim, offset_wrt_chunk, chunk,
+ size_m, sub_offset_m, buf);
+
+ /* Compress the chunk */
+ if (compressed &&
+ (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) &&
+ nbytes<chunk_size) {
+ outbuf = compressed;
+ } else {
+ outbuf = chunk;
+ nbytes = chunk_size;
+ }
+
+ /*
+ * Create the chunk it if it doesn't exist, or reallocate the chunk
+ * if its size changed. Then write the data into the file.
+ */
+ H5F_addr_undef(&(udata.addr));
+ udata.key.nbytes = nbytes;
+ if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+ "unable to allocate chunk");
+ }
+ if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) {
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "unable to write raw storage chunk");
+ }
+
+ /* Increment indices */
+ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) {
+ if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i];
+ else carry = 0;
+ }
+ if (carry) break;
+ }
+ ret_value = SUCCEED;
+
+ done:
+ H5MM_xfree(chunk);
+ H5MM_xfree (compressed);
+ FUNC_LEAVE(ret_value);
}
@@ -921,3 +1043,36 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ )
FUNC_LEAVE(SUCCEED);
}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_debug
+ *
+ * Purpose: Debugs a B-tree node for indexed raw data storage.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
+ intn fwidth, int ndims)
+{
+ H5F_istore_ud1_t udata;
+
+ FUNC_ENTER (H5F_istore_debug, FAIL);
+
+ HDmemset (&udata, 0, sizeof udata);
+ udata.mesg.ndims = ndims;
+
+ H5B_debug (f, addr, stream, indent, fwidth, H5B_ISTORE, &udata);
+
+ FUNC_LEAVE (SUCCEED);
+}
diff --git a/src/H5Flow.c b/src/H5Flow.c
index 5074815..faa2202 100644
--- a/src/H5Flow.c
+++ b/src/H5Flow.c
@@ -586,7 +586,7 @@ H5F_addr_cmp(const haddr_t *a1, const haddr_t *a2)
*-------------------------------------------------------------------------
*/
void
-H5F_addr_undef(haddr_t *addr /*out */ )
+H5F_addr_undef(haddr_t *addr/*out*/)
{
assert(addr);
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 3439686..d9ce06d 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -462,6 +462,7 @@ typedef struct H5F_t {
struct H5O_layout_t; /*forward decl for prototype arguments */
struct H5O_efl_t; /*forward decl for prototype arguments */
+struct H5O_compress_t; /*forward decl for prototype arguments */
/* library variables */
extern const H5F_create_t H5F_create_dflt;
@@ -476,14 +477,18 @@ H5F_t *H5F_open(const char *name, uintn flags,
herr_t H5F_close(H5F_t *f);
herr_t H5F_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
intn fwidth);
+herr_t H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream,
+ intn indent, intn fwidth, int ndims);
/* Functions that operate on array storage */
herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);
herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, const hsize_t _hslab_size[],
const hsize_t mem_size[], const hssize_t mem_offset[],
const hssize_t file_offset[], void *_buf/*out*/);
herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, const hsize_t _hslab_size[],
const hsize_t mem_size[], const hssize_t mem_offset[],
const hssize_t file_offset[], const void *_buf);
@@ -491,9 +496,11 @@ herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
/* Functions that operate on indexed storage */
herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);
herr_t H5F_istore_read(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const hssize_t offset[], const hsize_t size[],
void *buf /*out */ );
herr_t H5F_istore_write(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const hssize_t offset[], const hsize_t size[],
const void *buf);
diff --git a/src/H5Gnode.c b/src/H5Gnode.c
index 5580993..9b6b749 100644
--- a/src/H5Gnode.c
+++ b/src/H5Gnode.c
@@ -82,6 +82,7 @@ H5B_class_t H5B_SNODE[1] = {{
H5G_node_iterate, /*list */
H5G_node_decode_key, /*decode */
H5G_node_encode_key, /*encode */
+ NULL, /*debug key */
}};
/* Interface initialization */
diff --git a/src/H5MF.c b/src/H5MF.c
index 9d22c8a..fd5031b 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -128,3 +128,89 @@ H5MF_free(H5F_t __unused__ *f, const haddr_t *addr, hsize_t size)
FUNC_LEAVE(SUCCEED);
}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_realloc
+ *
+ * Purpose: Changes the size of an allocated chunk, possibly moving it to
+ * a new address. The chunk to change is at address ORIG_ADDR
+ * and is exactly ORIG_SIZE bytes (if these are zero and undef
+ * then this function acts like H5MF_alloc). The new size will
+ * be NEW_SIZE and its address is returned though NEW_ADDR (if
+ * NEW_SIZE is zero then this function acts like H5MF_free and
+ * an undefined address is returned for NEW_ADDR).
+ *
+ * If the new size is less than the old size then the new
+ * address will be the same as the old address (except for the
+ * special case where the new size is zero).
+ *
+ * If the new size is more than the old size then most likely a
+ * new address will be returned. However, under certain
+ * circumstances the library may return the same address.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_realloc (H5F_t *f, intn op, hsize_t orig_size, const haddr_t *orig_addr,
+ hsize_t new_size, haddr_t *new_addr/*out*/)
+{
+ FUNC_ENTER (H5MF_realloc, FAIL);
+
+ if (0==orig_size) {
+ /* Degenerate to H5MF_alloc() */
+ assert (!H5F_addr_defined (orig_addr));
+ if (new_size>0) {
+ if (H5MF_alloc (f, op, new_size, new_addr/*out*/)<0) {
+ HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL,
+ "unable to allocate new file memory");
+ }
+ } else {
+ H5F_addr_undef (new_addr);
+ }
+
+ } else if (0==new_size) {
+ /* Degenerate to H5MF_free() */
+ assert (H5F_addr_defined (orig_addr));
+ if (H5MF_free (f, orig_addr, orig_size)<0) {
+ HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL,
+ "unable to free old file memory");
+ }
+ H5F_addr_undef (new_addr);
+
+ } else if (new_size > orig_size) {
+ /* Size is getting larger */
+ if (H5MF_alloc (f, op, new_size, new_addr/*out*/)<0) {
+ HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL,
+ "unable to allocate new file memory");
+ }
+ if (H5MF_free (f, orig_addr, orig_size)<0) {
+ HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL,
+ "unable to free old file memory");
+ }
+
+ } else {
+ /* New size is not larger */
+#ifdef H5MF_DEBUG
+ if (new_size<orig_size) {
+ HDfprintf (stderr, "H5MF: realloc lost %Hd bytes\n",
+ orig_size-new_size);
+ }
+#endif
+ *new_addr = *orig_addr;
+ }
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+
diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h
index ea80ca1..202f36c 100644
--- a/src/H5MFprivate.h
+++ b/src/H5MFprivate.h
@@ -39,4 +39,8 @@
*/
herr_t H5MF_alloc (H5F_t *f, intn, hsize_t size, haddr_t *addr/*out*/);
herr_t H5MF_free (H5F_t *f, const haddr_t *addr, hsize_t size);
+herr_t H5MF_realloc (H5F_t *f, intn op, hsize_t orig_size,
+ const haddr_t *orig_addr, hsize_t new_size,
+ haddr_t *new_addr/*out*/);
+
#endif
diff --git a/src/H5MM.c b/src/H5MM.c
index c0e6d72..329a4f0 100644
--- a/src/H5MM.c
+++ b/src/H5MM.c
@@ -169,19 +169,8 @@ H5MM_xstrdup(const char *s)
*-------------------------------------------------------------------------
*/
void *
-H5MM_xfree(const void *mem)
+H5MM_xfree(void *mem)
{
- /*
- * free(3) takes a non-const pointer as an argument even though
- * conceptually the argument could be a constant because by time
- * free() mucks with it's contents, it should already be free :-)
- * Instead of passing a const arg to free, which generates a
- * compiler warning, we cast it to a non-const arg first. With
- * gcc, this results in a warning only if -Wcast-qual is turned on.
- */
- void *non_const_mem = (void *) mem;
-
- if (mem)
- HDfree(non_const_mem);
+ if (mem) HDfree(mem);
return NULL;
}
diff --git a/src/H5MMprivate.h b/src/H5MMprivate.h
index 3fef16b..2f715e5 100644
--- a/src/H5MMprivate.h
+++ b/src/H5MMprivate.h
@@ -29,5 +29,5 @@ void *H5MM_xmalloc (size_t size);
void *H5MM_xcalloc (intn n, size_t size);
void *H5MM_xrealloc (void *mem, size_t size);
char *H5MM_xstrdup (const char *s);
-void *H5MM_xfree (const void *mem);
+void *H5MM_xfree (void *mem);
#endif
diff --git a/src/H5O.c b/src/H5O.c
index b601809..c6b1c9d 100644
--- a/src/H5O.c
+++ b/src/H5O.c
@@ -61,7 +61,7 @@ static const H5O_class_t *const message_type_g[] = {
H5O_LAYOUT, /*0x0008 Data Layout */
NULL, /*0x0009 Not assigned */
NULL, /*0x000A Not assigned */
- NULL, /*0x000B Data storage -- compressed object */
+ H5O_COMPRESS, /*0x000B Data storage -- compressed object */
NULL, /*0x000C Attribute list */
H5O_NAME, /*0x000D Object name */
NULL, /*0x000E Object modification date and time */
diff --git a/src/H5Ocomp.c b/src/H5Ocomp.c
new file mode 100644
index 0000000..6b6c2e7
--- /dev/null
+++ b/src/H5Ocomp.c
@@ -0,0 +1,291 @@
+/*
+ * Copyright (C) 1998 NCSA
+ * All rights reserved.
+ *
+ * Programmer: Robb Matzke <matzke@llnl.gov>
+ * Wednesday, April 15, 1998
+ *
+ * Purpose: Data compression message.
+ */
+#include <H5private.h>
+#include <H5Eprivate.h>
+#include <H5MMprivate.h>
+#include <H5Oprivate.h>
+
+#define PABLO_MASK H5O_comp_mask
+
+/* PRIVATE PROTOTYPES */
+static herr_t H5O_comp_encode (H5F_t *f, uint8 *p, const void *mesg);
+static void *H5O_comp_decode (H5F_t *f, const uint8 *p, H5HG_t *hobj);
+static void *H5O_comp_copy (const void *_mesg, void *_dest);
+static size_t H5O_comp_size (H5F_t *f, const void *_mesg);
+static herr_t H5O_comp_reset (void *_mesg);
+static herr_t H5O_comp_debug (H5F_t *f, const void *_mesg,
+ FILE * stream, intn indent, intn fwidth);
+
+/* This message derives from H5O */
+const H5O_class_t H5O_COMPRESS[1] = {{
+ H5O_COMPRESS_ID, /* message id number */
+ "compression", /* message name for debugging */
+ sizeof(H5O_compress_t), /* native message size */
+ H5O_comp_decode, /* decode message */
+ H5O_comp_encode, /* encode message */
+ H5O_comp_copy, /* copy the native value */
+ H5O_comp_size, /* size of raw message */
+ H5O_comp_reset, /* reset method */
+ NULL, /* share method */
+ H5O_comp_debug, /* debug the message */
+}};
+
+/* Interface initialization */
+static hbool_t interface_initialize_g = FALSE;
+#define INTERFACE_INIT NULL
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_decode
+ *
+ * Purpose: Decodes a compression message.
+ *
+ * Return: Success: Ptr to the native message.
+ *
+ * Failure: NULL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5O_comp_decode(H5F_t __unused__ *f, const uint8 *p, H5HG_t __unused__ *hobj)
+{
+ H5O_compress_t *comp = NULL;
+
+ FUNC_ENTER(H5O_comp_decode, NULL);
+
+ /* check args */
+ assert(p);
+
+ /* Decode */
+ comp = H5MM_xcalloc(1, sizeof *comp);
+ comp->method = *p++;
+ comp->flags = *p++;
+ UINT16DECODE (p, comp->cd_size);
+
+ if (comp->cd_size>0) {
+ comp->client_data = H5MM_xmalloc (comp->cd_size);
+ HDmemcpy (comp->client_data, p, comp->cd_size);
+ }
+
+ FUNC_LEAVE(comp);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_encode
+ *
+ * Purpose: Encodes message MESG into buffer P.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5O_comp_encode (H5F_t __unused__ *f, uint8 *p/*out*/, const void *mesg)
+{
+ const H5O_compress_t *comp = (const H5O_compress_t*)mesg;
+
+ FUNC_ENTER (H5O_comp_encode, FAIL);
+
+ /* Check args */
+ assert (p);
+ assert (mesg);
+
+ *p++ = comp->method;
+ *p++ = comp->flags;
+ UINT16ENCODE (p, comp->cd_size);
+ if (comp->cd_size) {
+ HDmemcpy (p, comp->client_data, comp->cd_size);
+ }
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_copy
+ *
+ * Purpose: Copies a compression message from SRC to DST allocating DST
+ * if necessary. If DST is already allocated then we assume
+ * that it isn't initialized.
+ *
+ * Return: Success: Ptr to DST or allocated result.
+ *
+ * Failure: NULL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5O_comp_copy (const void *_src, void *_dst/*out*/)
+{
+ const H5O_compress_t *src = (const H5O_compress_t *)_src;
+ H5O_compress_t *dst = (H5O_compress_t *)_dst;
+
+ FUNC_ENTER (H5O_comp_copy, NULL);
+
+ if (!dst) dst = H5MM_xmalloc (sizeof *dst);
+ *dst = *src;
+ if (src->cd_size>0) {
+ dst->client_data = H5MM_xmalloc (src->cd_size);
+ HDmemcpy (dst->client_data, src->client_data, src->cd_size);
+ }
+
+ FUNC_LEAVE (dst);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_size
+ *
+ * Purpose: Determines the size of a raw compression message.
+ *
+ * Return: Success: Size of message.
+ *
+ * Failure: zero
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+H5O_comp_size (H5F_t __unused__ *f, const void *mesg)
+{
+ const H5O_compress_t *comp = (const H5O_compress_t*)mesg;
+
+ FUNC_ENTER (H5O_comp_size, 0);
+ FUNC_LEAVE (4+comp->cd_size);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_reset
+ *
+ * Purpose: Resets a compression message by freeing the client data and
+ * setting all fields to zero. The MESG buffer is not freed.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5O_comp_reset (void *mesg)
+{
+ H5O_compress_t *comp = (H5O_compress_t*)mesg;
+
+ FUNC_ENTER (H5O_comp_reset, FAIL);
+
+ assert (comp);
+ H5MM_xfree (comp->client_data);
+ HDmemset (comp, 0, sizeof *comp);
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_comp_debug
+ *
+ * Purpose: Prints debugging information for compression message MESG on
+ * output stream STREAM. Each line is indented INDENT
+ * characters and the field name takes up FWIDTH characters.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5O_comp_debug (H5F_t __unused__ *f, const void *mesg, FILE *stream,
+ intn indent, intn fwidth)
+{
+ const H5O_compress_t *comp = (const H5O_compress_t *)mesg;
+ size_t i, j;
+
+ FUNC_ENTER(H5O_comp_debug, FAIL);
+
+ /* check args */
+ assert(f);
+ assert(comp);
+ assert(stream);
+ assert(indent >= 0);
+ assert(fwidth >= 0);
+
+ fprintf (stream, "%*s%-*s %d\n", indent, "", fwidth,
+ "Method:",
+ (int)(comp->method));
+ fprintf (stream, "%*s%-*s 0x%02x\n", indent, "", fwidth,
+ "Flags:",
+ (unsigned)(comp->flags));
+ fprintf (stream, "%*s%-*s %lu\n", indent, "", fwidth,
+ "Size of client data:",
+ (unsigned long)(comp->cd_size));
+
+ if (comp->cd_size>0) {
+ fprintf (stream, "%*s%s\n", indent, "", "Client Data:");
+ for (i=0; i<comp->cd_size; i+=16) {
+ fprintf (stream, "%*s%04d: ", indent+3, "", i);
+ for (j=0; j<16; j++) {
+ if (8==j) putc (' ', stream);
+ if (i+j<comp->cd_size) {
+ fprintf (stream, "%02x ", comp->client_data[i+j]);
+ } else {
+ fputs (" ", stream);
+ }
+ }
+ for (j=0; j<16 && i+j<comp->cd_size; j++) {
+ if (8==j) putc (' ', stream);
+ if (comp->client_data[i+j]>' ' &&
+ comp->client_data[i+j]<='~') {
+ putc (comp->client_data[i+j], stream);
+ } else {
+ putc ('.', stream);
+ }
+ putc ('\n', stream);
+ }
+ }
+ } else {
+ fprintf (stream, "%*s%-*s None\n", indent, "", fwidth, "Client Data:");
+ }
+
+ FUNC_LEAVE(SUCCEED);
+}
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 7f4fd63..d3836f4 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -26,6 +26,7 @@
#include <H5HGprivate.h>
#include <H5Tprivate.h>
#include <H5Sprivate.h>
+#include <H5Zprivate.h>
/*
* Align messages on 8-byte boundaries because we would like to copy the
@@ -166,6 +167,19 @@ typedef struct H5O_layout_t {
} H5O_layout_t;
/*
+ * Compression message.
+ */
+#define H5O_COMPRESS_ID 0x000b
+extern const H5O_class_t H5O_COMPRESS[1];
+
+typedef struct H5O_compress_t {
+ H5Z_method_t method; /*algorithm ID */
+ uintn flags; /*flags */
+ size_t cd_size; /*client data size */
+ uint8 *client_data; /*client data passed to algorithm */
+} H5O_compress_t;
+
+/*
* Object name message.
*/
#define H5O_NAME_ID 0x000d
diff --git a/src/H5P.c b/src/H5P.c
index 2d0e6f0..b141853 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -942,7 +942,7 @@ H5Pget_chunk(hid_t tid, int max_ndims, hsize_t dim[]/*out*/)
if (H5P_DATASET_CREATE != H5Pget_class(tid) ||
NULL == (tmpl = H5I_object(tid))) {
HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL,
- "not a dataset creation template");
+ "not a dataset creation property list");
}
if (H5D_CHUNKED != tmpl->layout) {
HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
@@ -1847,7 +1847,178 @@ H5Pget_preserve (hid_t plist_id)
FUNC_LEAVE (plist->need_bkg?TRUE:FALSE);
}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_compression
+ *
+ * Purpose: Sets the compression method in a dataset creation property
+ * list. This sets default values for the compression
+ * attributes by setting the flags to zero and supplying no
+ * compression client data. It's probably better to use
+ * specific compression initialization functions like
+ * H5Pset_deflate().
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_compression (hid_t plist_id, H5Z_method_t method)
+{
+ H5D_create_t *plist = NULL;
+
+ FUNC_ENTER (H5Pset_compression, FAIL);
+
+ /* Check arguments */
+ if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) ||
+ NULL==(plist=H5I_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset creation property list");
+ }
+ if (method<0 || method>H5Z_MAXVAL) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid compression method");
+ }
+
+ /* Clear any previous compression method info, then set new value */
+ H5O_reset (H5O_COMPRESS, &(plist->compress));
+ plist->compress.method = method;
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_compression
+ *
+ * Purpose: Gets the compression method information from a dataset
+ * creation property list.
+ *
+ * Return: Success: Compression method.
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+H5Z_method_t
+H5Pget_compression (hid_t plist_id)
+{
+ H5D_create_t *plist = NULL;
+
+ FUNC_ENTER (H5Pget_compression, FAIL);
+
+ /* Check arguments */
+ if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) ||
+ NULL==(plist=H5I_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset creation property list");
+ }
+
+ FUNC_LEAVE (plist->compress.method);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_deflate
+ *
+ * Purpose: Sets the compression method for a dataset creation property
+ * list to H5D_COMPRESS_DEFLATE and the compression level to
+ * LEVEL which should be a value between zero and nine,
+ * inclusive. Lower compression levels are faster but result in
+ * less compression. This is the same algorithm as used by the
+ * GNU gzip program.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_deflate (hid_t plist_id, int level)
+{
+ H5D_create_t *plist = NULL;
+
+ FUNC_ENTER (H5Pset_deflate, FAIL);
+
+ /* Check arguments */
+ if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) ||
+ NULL==(plist=H5I_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset creation property list");
+ }
+ if (level<0 || level>9) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid deflate level");
+ }
+
+ /* Clear previous compression parameters */
+ H5O_reset (H5O_COMPRESS, &(plist->compress));
+ plist->compress.method = H5Z_DEFLATE;
+ plist->compress.flags = level;
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_deflate
+ *
+ * Purpose: Returns the deflate compression level from a dataset creation
+ * property list that uses that method.
+ *
+ * Return: Success: A value between zero and nine, inclusive.
+ * Smaller values indicate faster compression
+ * while higher values indicate better
+ * compression ratios.
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+H5Pget_deflate (hid_t plist_id)
+{
+ H5D_create_t *plist = NULL;
+
+ FUNC_ENTER (H5Pget_deflate, FAIL);
+ /* Check arguments */
+ if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) ||
+ NULL==(plist=H5I_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset creation property list");
+ }
+ if (H5Z_DEFLATE!=plist->compress.method) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "deflate method is not being used");
+ }
+
+ FUNC_LEAVE (plist->compress.flags % 10);
+}
#ifdef HAVE_PARALLEL
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index ad7f98b..bd87342 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -25,6 +25,7 @@
#include <H5Ipublic.h>
#include <H5Dpublic.h>
#include <H5Fpublic.h>
+#include <H5Zpublic.h>
/* Template classes */
typedef enum H5P_class_t {
@@ -87,6 +88,10 @@ herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
herr_t H5Pset_preserve (hid_t plist_id, hbool_t status);
int H5Pget_preserve (hid_t plist_id);
+herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method);
+H5Z_method_t H5Pget_compression (hid_t plist_id);
+herr_t H5Pset_deflate (hid_t plist_id, int level);
+int H5Pget_deflate (hid_t plist_id);
#ifdef HAVE_PARALLEL
herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info);
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index db6bf9e..07f4108 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -70,6 +70,7 @@ typedef struct H5S_tconv_t {
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
@@ -86,18 +87,21 @@ typedef struct H5S_tconv_t {
/* Scatter elements from type conversion buffer to disk */
herr_t (*fscat)(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *compress,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, const void *tconv_buf);
/* Read from file to application w/o intermediate scratch buffer */
herr_t (*read)(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_t *mem_space,
void *buf/*out*/);
/* Write directly from app buffer to file */
herr_t (*write)(H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_t *mem_space,
const void *buf);
@@ -125,6 +129,7 @@ size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
size_t desired_nelmts, H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
@@ -135,14 +140,17 @@ size_t H5S_simp_mgath (const void *buf, size_t elmt_size,
const H5S_t *mem_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, const void *tconv_buf);
herr_t H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_t *mem_space,
void *buf/*out*/);
herr_t H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_compress_t *comp,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_t *mem_space,
const void *buf);
diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c
index 976840f..f8b3e0d 100644
--- a/src/H5Ssimp.c
+++ b/src/H5Ssimp.c
@@ -124,7 +124,7 @@ H5S_simp_init (const struct H5O_layout_t __unused__ *layout,
*/
size_t
H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl,
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t __unused__ *numbering,
size_t start, size_t nelmts, void *buf/*out*/)
@@ -184,7 +184,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Gather from file.
*/
- if (H5F_arr_read (f, layout, efl, hsize, hsize, zero, file_offset,
+ if (H5F_arr_read (f, layout, comp, efl, hsize, hsize, zero, file_offset,
buf/*out*/)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_READERROR, 0, "read error");
}
@@ -408,7 +408,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
*/
herr_t
H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl,
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t __unused__ *numbering,
size_t start, size_t nelmts, const void *buf)
@@ -468,7 +468,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Scatter to file.
*/
- if (H5F_arr_write (f, layout, efl, hsize, hsize, zero,
+ if (H5F_arr_write (f, layout, comp, efl, hsize, hsize, zero,
file_offset, buf)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error");
}
@@ -499,9 +499,9 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
*/
herr_t
H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl, size_t elmt_size,
- const H5S_t *file_space, const H5S_t *mem_space,
- void *buf/*out*/)
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
+ size_t elmt_size, const H5S_t *file_space,
+ const H5S_t *mem_space, void *buf/*out*/)
{
hssize_t file_offset[H5O_LAYOUT_NDIMS];
hsize_t hslab_size[H5O_LAYOUT_NDIMS];
@@ -573,7 +573,7 @@ H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout,
mem_offset[file_space->u.simple.rank] = 0;
/* Read the hyperslab */
- if (H5F_arr_read (f, layout, efl, hslab_size,
+ if (H5F_arr_read (f, layout, comp, efl, hslab_size,
mem_size, mem_offset, file_offset, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, "unable to read dataset");
}
@@ -604,9 +604,9 @@ H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout,
*/
herr_t
H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout,
- const struct H5O_efl_t *efl, size_t elmt_size,
- const H5S_t *file_space, const H5S_t *mem_space,
- const void *buf)
+ const struct H5O_compress_t *comp, const struct H5O_efl_t *efl,
+ size_t elmt_size, const H5S_t *file_space,
+ const H5S_t *mem_space, const void *buf)
{
hssize_t file_offset[H5O_LAYOUT_NDIMS];
hsize_t hslab_size[H5O_LAYOUT_NDIMS];
@@ -678,7 +678,7 @@ H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout,
mem_offset[file_space->u.simple.rank] = 0;
/* Write the hyperslab */
- if (H5F_arr_write (f, layout, efl, hslab_size,
+ if (H5F_arr_write (f, layout, comp, efl, hslab_size,
mem_size, mem_offset, file_offset, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"unable to write dataset");
diff --git a/src/H5T.c b/src/H5T.c
index c13441b..488e30d 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -231,15 +231,17 @@ H5T_init_interface(void)
* Register conversion functions beginning with the most general and
* ending with the most specific.
*/
- if (H5Tregister_soft(H5T_INTEGER, H5T_INTEGER, H5T_conv_order) < 0) {
+ if (H5Tregister_soft("ibo", H5T_INTEGER, H5T_INTEGER,
+ H5T_conv_order) < 0) {
HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
"unable to register conversion function");
}
- if (H5Tregister_soft(H5T_FLOAT, H5T_FLOAT, H5T_conv_order) < 0) {
+ if (H5Tregister_soft("fbo", H5T_FLOAT, H5T_FLOAT, H5T_conv_order) < 0) {
HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
"unable to register conversion function");
}
- if (H5Tregister_soft (H5T_COMPOUND, H5T_COMPOUND, H5T_conv_struct)<0) {
+ if (H5Tregister_soft ("struct", H5T_COMPOUND, H5T_COMPOUND,
+ H5T_conv_struct)<0) {
HRETURN_ERROR (H5E_DATATYPE, H5E_CANTINIT, FAIL,
"unable to register conversion function");
}
@@ -268,10 +270,14 @@ H5T_init_interface(void)
static void
H5T_term_interface(void)
{
- int i;
+ intn i;
H5T_path_t *path = NULL;
H5T_cdata_t *pcdata = NULL;
H5T_conv_t cfunc = NULL;
+#ifdef H5T_DEBUG
+ intn nprint=0;
+ hsize_t nbytes;
+#endif
/* Unregister all conversion functions */
for (i=0; i<H5T_npath_g; i++) {
@@ -286,6 +292,29 @@ H5T_term_interface(void)
#endif
H5E_clear(); /*ignore the error*/
}
+#ifdef H5T_DEBUG
+ if (0==nprint++) {
+ HDfprintf (stderr, "H5T: type conversion statistics "
+ "accumulated over life of library:\n");
+ HDfprintf (stderr, " %-*s %8s/%-5s %8s %8s %8s %15s\n",
+ H5T_NAMELEN-1, "Name", "Elmts", "Calls", "User",
+ "System", "Elapsed", "Bandwidth");
+ HDfprintf (stderr, " %-*s %8s-%-5s %8s %8s %8s %15s\n",
+ H5T_NAMELEN-1, "----", "-----", "-----", "----",
+ "------", "-------", "---------");
+ }
+ nbytes = MAX (H5T_get_size (path->src), H5T_get_size (path->dst));
+ nbytes *= path->cdata.stats->nelmts;
+ HDfprintf (stderr, " %-*s %8Hd/%-5d %8.2f %8.2f %8.2f %15g\n",
+ H5T_NAMELEN-1, path->name,
+ path->cdata.stats->nelmts,
+ path->cdata.stats->ncalls,
+ path->cdata.stats->timer.utime,
+ path->cdata.stats->timer.stime,
+ path->cdata.stats->timer.etime,
+ nbytes / path->cdata.stats->timer.etime);
+#endif
+ H5MM_xfree (path->cdata.stats);
}
}
@@ -2276,7 +2305,7 @@ H5Tis_shared (hid_t loc_id, hid_t type_id)
*-------------------------------------------------------------------------
*/
herr_t
-H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func)
+H5Tregister_hard(const char *name, hid_t src_id, hid_t dst_id, H5T_conv_t func)
{
H5T_t *src = NULL;
H5T_t *dst = NULL;
@@ -2286,6 +2315,10 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func)
FUNC_ENTER(H5Tregister_hard, FAIL);
/* Check args */
+ if (!name || !*name) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "conversion must have a name for debugging");
+ }
if (H5_DATATYPE != H5I_group(src_id) ||
NULL == (src = H5I_object(src_id)) ||
H5_DATATYPE != H5I_group(dst_id) ||
@@ -2294,7 +2327,7 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func)
}
/* Locate or create a new conversion path */
- if (NULL == (path = H5T_path_find(src, dst, TRUE, func))) {
+ if (NULL == (path = H5T_path_find(name, src, dst, TRUE, func))) {
HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
"unable to locate/allocate conversion path");
}
@@ -2334,7 +2367,8 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func)
*-------------------------------------------------------------------------
*/
herr_t
-H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func)
+H5Tregister_soft(const char *name, H5T_class_t src_cls, H5T_class_t dst_cls,
+ H5T_conv_t func)
{
intn i;
hid_t src_id, dst_id;
@@ -2343,6 +2377,10 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func)
FUNC_ENTER(H5Tregister_soft, FAIL);
/* Check args */
+ if (!name || !*name) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "conversion must have a name for debugging");
+ }
if (src_cls < 0 || src_cls >= H5T_NCLASSES ||
dst_cls < 0 || dst_cls >= H5T_NCLASSES) {
HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
@@ -2359,6 +2397,8 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func)
H5T_soft_g = H5MM_xrealloc(H5T_soft_g,
H5T_asoft_g * sizeof(H5T_soft_t));
}
+ HDstrncpy (H5T_soft_g[H5T_nsoft_g].name, name, H5T_NAMELEN);
+ H5T_soft_g[H5T_nsoft_g].name[H5T_NAMELEN-1] = '\0';
H5T_soft_g[H5T_nsoft_g].src = src_cls;
H5T_soft_g[H5T_nsoft_g].dst = dst_cls;
H5T_soft_g[H5T_nsoft_g].func = func;
@@ -2387,6 +2427,7 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func)
HDmemset (&cdata, 0, sizeof cdata);
cdata.command = H5T_CONV_INIT;
+ cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t));
if ((func) (src_id, dst_id, &cdata, 0, NULL, NULL) >= 0) {
/*
* Free resources used by the previous conversion function. We
@@ -2403,9 +2444,14 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func)
#endif
H5E_clear();
}
+ H5MM_xfree (path->cdata.stats);
}
+ HDstrncpy (path->name, name, H5T_NAMELEN);
+ path->name[H5T_NAMELEN-1] = '\0';
path->func = func;
path->cdata = cdata;
+ } else {
+ H5MM_xfree (cdata.stats);
}
/* Release temporary atoms */
@@ -2475,6 +2521,7 @@ H5Tunregister(H5T_conv_t func)
#endif
H5E_clear();
}
+ H5MM_xfree (path->cdata.stats);
HDmemset (&(path->cdata), 0, sizeof(H5T_cdata_t));
/*
@@ -2500,8 +2547,10 @@ H5Tunregister(H5T_conv_t func)
}
path->cdata.command = H5T_CONV_INIT;
+ path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t));
if ((H5T_soft_g[j].func)(src_id, dst_id, &(path->cdata),
0, NULL, NULL) >= 0) {
+ HDstrcpy (path->name, H5T_soft_g[j].name);
path->func = H5T_soft_g[j].func;
} else {
H5E_clear();
@@ -3348,6 +3397,9 @@ H5T_cmp(const H5T_t *dt1, const H5T_t *dt2)
* H5T_BKG_YES then a pointer to the H5T_conv_noop() function is
* returned.
*
+ * NAME is assigned to the conversion path if the path is
+ * created. The name is only for debugging.
+ *
* Return: Success: A pointer to an appropriate conversion
* function. The PCDATA argument is initialized
* to point to type conversion data which should
@@ -3378,9 +3430,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg,
HRETURN(H5T_conv_noop);
}
-
/* Find it */
- if (NULL == (path = H5T_path_find(src, dst, TRUE, NULL))) {
+ if (NULL == (path = H5T_path_find(NULL, src, dst, TRUE, NULL))) {
HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL,
"unable to create conversion path");
}
@@ -3403,6 +3454,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg,
* is created. If FUNC is non-null then it is registered as the
* hard function for that path.
*
+ * If a path is created then NAME is used for debugging.
+ *
* Return: Success: Pointer to the path, valid until the path
* database is modified.
*
@@ -3416,8 +3469,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg,
*-------------------------------------------------------------------------
*/
H5T_path_t *
-H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create,
- H5T_conv_t func)
+H5T_path_find(const char *name, const H5T_t *src, const H5T_t *dst,
+ hbool_t create, H5T_conv_t func)
{
intn lt = 0; /*left edge (inclusive) */
intn rt = H5T_npath_g; /*right edge (exclusive) */
@@ -3471,9 +3524,12 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create,
/* Associate a function with the path if possible */
if (func) {
+ HDstrncpy (path->name, name, H5T_NAMELEN);
+ path->name[H5T_NAMELEN-1] = '\0';
path->func = func;
path->is_hard = TRUE;
path->cdata.command = H5T_CONV_INIT;
+ path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t));
if ((src_id=H5I_register(H5_DATATYPE, H5T_copy(path->src))) < 0 ||
(dst_id=H5I_register(H5_DATATYPE, H5T_copy(path->dst))) < 0) {
HRETURN_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL,
@@ -3503,11 +3559,13 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create,
"unable to register conv types for query");
}
path->cdata.command = H5T_CONV_INIT;
+ path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t));
if ((H5T_soft_g[i].func) (src_id, dst_id, &(path->cdata),
H5T_CONV_INIT, NULL, NULL) < 0) {
HDmemset (&(path->cdata), 0, sizeof(H5T_cdata_t));
H5E_clear(); /*ignore the error*/
} else {
+ HDstrcpy (path->name, H5T_soft_g[i].name);
path->func = H5T_soft_g[i].func;
}
H5I_dec_ref(src_id);
@@ -3518,6 +3576,59 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create,
FUNC_LEAVE(path);
}
+
+/*-------------------------------------------------------------------------
+ * Function: H5T_timer_begin
+ *
+ * Purpose: Start a timer for a data type conversion.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Friday, April 17, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5T_timer_begin (H5_timer_t *timer, H5T_cdata_t *cdata)
+{
+ assert (timer);
+ assert (cdata);
+ assert (cdata->stats);
+
+ H5_timer_begin (timer);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5T_timer_end
+ *
+ * Purpose: Ends a timer for a data type conversion
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Friday, April 17, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5T_timer_end (H5_timer_t *timer, H5T_cdata_t *cdata, size_t nelmts)
+{
+ assert (timer);
+ assert (cdata);
+ assert (cdata->stats);
+
+ H5_timer_end (&(cdata->stats->timer), timer);
+ cdata->stats->ncalls++;
+ cdata->stats->nelmts += nelmts;
+}
+
+
/*-------------------------------------------------------------------------
* Function: H5T_debug
*
diff --git a/src/H5Tconv.c b/src/H5Tconv.c
index 0e53d18..be07341 100644
--- a/src/H5Tconv.c
+++ b/src/H5Tconv.c
@@ -62,14 +62,8 @@ H5T_conv_noop(hid_t __unused__ src_id, hid_t __unused__ dst_id,
case H5T_CONV_FREE:
/* Nothing to free */
-#ifdef H5T_DEBUG
- if (cdata->ncalls>0) {
- fprintf (stderr, "H5T: H5T_conv_noop statistics...\n");
- fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls);
- fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts);
- }
-#endif
-
+ break;
+
default:
HRETURN_ERROR (H5E_DATATYPE, H5E_UNSUPPORTED, FAIL,
"unknown conversion command");
@@ -173,13 +167,6 @@ H5T_conv_order(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
case H5T_CONV_FREE:
/* Free private data */
-#ifdef H5T_DEBUG
- if (cdata->ncalls>0) {
- fprintf (stderr, "H5T: H5T_conv_order statistics...\n");
- fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls);
- fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts);
- }
-#endif
break;
default:
@@ -396,13 +383,6 @@ H5T_conv_struct(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
/*
* Free the private conversion data.
*/
-#ifdef H5T_DEBUG
- if (cdata->ncalls>0) {
- fprintf (stderr, "H5T: H5T_conv_struct statistics...\n");
- fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls);
- fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts);
- }
-#endif
H5MM_xfree (priv->src2dst);
H5MM_xfree (priv->src_memb_id);
H5MM_xfree (priv->dst_memb_id);
diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h
index e6eed20..64d0fba 100644
--- a/src/H5Tpkg.h
+++ b/src/H5Tpkg.h
@@ -26,6 +26,8 @@
#include <H5HGprivate.h>
#include <H5Tprivate.h>
+#define H5T_NAMELEN 10 /*length of debugging name buffer */
+
typedef struct H5T_atomic_t {
H5T_order_t order; /*byte order */
size_t prec; /*precision in bits */
@@ -84,6 +86,7 @@ typedef struct H5T_member_t {
/* The data type conversion database */
typedef struct H5T_path_t {
+ char name[H5T_NAMELEN]; /*name for debugging only */
H5T_t *src; /*source data type ID */
H5T_t *dst; /*destination data type ID */
H5T_conv_t func; /*data conversion function */
@@ -93,14 +96,24 @@ typedef struct H5T_path_t {
/* The master list of soft conversion functions */
typedef struct H5T_soft_t {
+ char name[H5T_NAMELEN]; /*name for debugging only */
H5T_class_t src; /*source data type class */
H5T_class_t dst; /*destination data type class */
H5T_conv_t func; /*the conversion function */
} H5T_soft_t;
+/* Statistics about a conversion function */
+typedef struct H5T_stats_t {
+ uintn ncalls; /*num calls to conversion function */
+#ifdef H5T_DEBUG
+ hsize_t nelmts; /*total data points converted */
+ H5_timer_t timer; /*total time for conversion */
+#endif
+} H5T_stats_t;
+
/* Function prototypes for H5T package scope */
-H5T_path_t *H5T_path_find (const H5T_t *src, const H5T_t *dst,
- hbool_t create, H5T_conv_t func);
+H5T_path_t *H5T_path_find (const char *name, const H5T_t *src,
+ const H5T_t *dst, hbool_t create, H5T_conv_t func);
/* Conversion functions */
herr_t H5T_conv_order (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h
index 388c20a..a9241d6 100644
--- a/src/H5Tprivate.h
+++ b/src/H5Tprivate.h
@@ -45,6 +45,8 @@ herr_t H5T_pack (H5T_t *dt);
herr_t H5T_debug (H5T_t *dt, FILE * stream);
H5T_conv_t H5T_find (const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg,
H5T_cdata_t **pcdata/*out*/);
+void H5T_timer_begin (H5_timer_t *timer, H5T_cdata_t *cdata);
+void H5T_timer_end (H5_timer_t *timer, H5T_cdata_t *cdata, size_t nelmts);
/*
* This conversion function is here so we can determine whether a conversion
diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h
index c17ac3a..4c1a680 100644
--- a/src/H5Tpublic.h
+++ b/src/H5Tpublic.h
@@ -111,9 +111,8 @@ typedef struct H5T_cdata_t {
H5T_cmd_t command;/*what should the conversion function do? */
H5T_bkg_t need_bkg;/*is the background buffer needed? */
hbool_t recalc; /*recalculate private data */
- unsigned long ncalls; /*number of calls to conversion function */
- unsigned long nelmts; /*total number of data points converted */
void *priv; /*private data */
+ struct H5T_stats_t *stats; /*statistics for the conversion */
} H5T_cdata_t;
/* All data type conversion functions are... */
@@ -232,8 +231,10 @@ herr_t H5Tset_cset (hid_t type_id, H5T_cset_t cset);
herr_t H5Tset_strpad (hid_t type_id, H5T_str_t strpad);
/* Type conversion database */
-herr_t H5Tregister_hard (hid_t src_id, hid_t dst_id, H5T_conv_t func);
-herr_t H5Tregister_soft (H5T_class_t src, H5T_class_t dst, H5T_conv_t func);
+herr_t H5Tregister_hard (const char *name, hid_t src_id, hid_t dst_id,
+ H5T_conv_t func);
+herr_t H5Tregister_soft (const char *name, H5T_class_t src, H5T_class_t dst,
+ H5T_conv_t func);
herr_t H5Tunregister (H5T_conv_t func);
H5T_conv_t H5Tfind (hid_t src_id, hid_t dst_id, H5T_cdata_t **pcdata);
diff --git a/src/H5Z.c b/src/H5Z.c
new file mode 100644
index 0000000..993cc5e
--- /dev/null
+++ b/src/H5Z.c
@@ -0,0 +1,501 @@
+/*
+ * Copyright (C) 1998 NCSA
+ * All rights reserved.
+ *
+ * Programmer: Robb Matzke <matzke@llnl.gov>
+ * Thursday, April 16, 1998
+ *
+ * Purpose: Functions for data compression.
+ */
+#include <H5private.h>
+#include <H5Eprivate.h>
+#include <H5MMprivate.h>
+#include <H5Oprivate.h>
+#include <H5Zprivate.h>
+
+#ifdef HAVE_ZLIB_H
+# include <zlib.h>
+#endif
+
+/* Interface initialization */
+#define PABLO_MASK H5Z_mask
+#define INTERFACE_INIT H5Z_init_interface
+static intn interface_initialize_g = FALSE;
+static herr_t H5Z_init_interface (void);
+static void H5Z_term_interface (void);
+
+/*
+ * The compression table maps compression method number to a struct that
+ * contains pointers to the compress and uncompress methods along with timing
+ * statistics.
+ */
+typedef struct H5Z_class_t {
+ char *name; /*method name for debugging */
+ H5Z_func_t compress; /*compression function */
+ H5Z_func_t uncompress; /*uncompression function */
+
+#ifdef H5Z_DEBUG
+ struct {
+ hsize_t nbytes; /*bytes compressed including overruns */
+ hsize_t over; /*bytes of overrun */
+ H5_timer_t timer; /*total compression time inc. overruns */
+ hsize_t failed; /*bytes of failure (not overruns) */
+ } comp;
+
+ struct {
+ hsize_t nbytes; /*bytes uncompressed, including overruns*/
+ hsize_t over; /*bytes of overrun */
+ H5_timer_t timer; /*total uncompression time */
+ hsize_t failed; /*bytes of failure (not overruns) */
+ } uncomp;
+#endif
+} H5Z_class_t;
+static H5Z_class_t H5Z_g[H5Z_MAXVAL+1];
+
+/* Compression and uncompression methods */
+static size_t H5Z_zlib_c (unsigned int flags, size_t __unused__ cd_size,
+ const void __unused__ *client_data,
+ size_t src_nbytes, const void *_src,
+ size_t dst_nbytes, void *dst/*out*/);
+static size_t H5Z_zlib_u (unsigned int flags, size_t __unused__ cd_size,
+ const void __unused__ *client_data,
+ size_t src_nbytes, const void *_src,
+ size_t dst_nbytes, void *dst/*out*/);
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_init_interface
+ *
+ * Purpose: Initializes the data compression layer.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5Z_init_interface (void)
+{
+ FUNC_ENTER (H5Z_init_interface, FAIL);
+
+ H5_add_exit (H5Z_term_interface);
+
+ H5Z_register (H5Z_NONE, "none", NULL, NULL);
+ H5Z_register (H5Z_DEFLATE, "deflate", H5Z_zlib_c, H5Z_zlib_u);
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_term_interface
+ *
+ * Purpose: Terminate the H5Z layer.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void
+H5Z_term_interface (void)
+{
+#ifdef H5Z_DEBUG
+ int i, nprint=0;
+ char name[16];
+
+ for (i=0; i<=H5Z_MAXVAL; i++) {
+ if (H5Z_g[i].comp.nbytes || H5Z_g[i].uncomp.nbytes) {
+ if (0==nprint++) {
+ HDfprintf (stderr, "H5Z: compression statistics accumulated "
+ "over life of library:\n");
+ HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n",
+ "Method", "Total", "Overrun", "Errors", "User",
+ "System", "Elapsed", "Bandwidth");
+ HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n",
+ "------", "-----", "-------", "------", "----",
+ "------", "-------", "---------");
+ }
+ sprintf (name, "%s-c", H5Z_g[i].name);
+ HDfprintf (stderr,
+ " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ",
+ name,
+ H5Z_g[i].comp.nbytes,
+ H5Z_g[i].comp.over,
+ H5Z_g[i].comp.failed,
+ H5Z_g[i].comp.timer.utime,
+ H5Z_g[i].comp.timer.stime,
+ H5Z_g[i].comp.timer.etime);
+ if (H5Z_g[i].comp.timer.etime>0) {
+ HDfprintf (stderr, "%9.3e\n",
+ H5Z_g[i].comp.nbytes / H5Z_g[i].comp.timer.etime);
+ } else {
+ HDfprintf (stderr, "%9s\n", "NaN");
+ }
+
+ sprintf (name, "%s-u", H5Z_g[i].name);
+ HDfprintf (stderr,
+ " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ",
+ name,
+ H5Z_g[i].uncomp.nbytes,
+ H5Z_g[i].uncomp.over,
+ H5Z_g[i].uncomp.failed,
+ H5Z_g[i].uncomp.timer.utime,
+ H5Z_g[i].uncomp.timer.stime,
+ H5Z_g[i].uncomp.timer.etime);
+ if (H5Z_g[i].uncomp.timer.etime>0) {
+ HDfprintf (stderr, "%9.3e\n",
+ H5Z_g[i].uncomp.nbytes/H5Z_g[i].uncomp.timer.etime);
+ } else {
+ HDfprintf (stderr, "%9s\n", "NaN");
+ }
+ }
+ }
+#endif
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Zregister
+ *
+ * Purpose: This function registers new compression and uncompression
+ * methods for a method number. The NAME argument is used for
+ * debugging and may be the null pointer. Either or both of
+ * CFUNC (the compression function) and UFUNC (the uncompression
+ * method) may be null pointers.
+ *
+ * The statistics associated with a method number are not reset
+ * by this function; they accumulate over the life of the
+ * library.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Zregister (H5Z_method_t method, const char *name, H5Z_func_t cfunc,
+ H5Z_func_t ufunc)
+{
+ FUNC_ENTER (H5Zregister, FAIL);
+
+ /* Check args */
+ if (method<0 || method>H5Z_MAXVAL) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid data compression method number");
+ }
+ if (method<16) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to modify predefined compression methods");
+ }
+
+ /* Do it */
+ if (H5Z_register (method, name, cfunc, ufunc)<0) {
+ HRETURN_ERROR (H5E_COMP, H5E_CANTINIT, FAIL,
+ "unable to register compression methods");
+ }
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_compress
+ *
+ * Purpose: Compress NBYTES from SRC into at most NBYTES of DST.
+ *
+ * Return: Success: Number of bytes in DST
+ *
+ * Failure: 0 if the DST buffer overflowed or something
+ * else went wrong.
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Z_compress (const H5O_compress_t *comp, size_t nbytes, const void *src,
+ void *dst/*out*/)
+{
+ size_t ret_value = 0;
+ H5Z_method_t method = comp ? comp->method : H5Z_NONE;
+ H5Z_func_t cfunc = NULL;
+#ifdef H5Z_DEBUG
+ intn over = 0;
+ H5_timer_t timer;
+#endif
+
+ FUNC_ENTER (H5Z_compress, 0);
+
+#ifdef H5Z_DEBUG
+ H5_timer_begin (&timer);
+#endif
+
+ if (H5Z_NONE==method) {
+ /* No compression method */
+ HGOTO_DONE (0);
+
+ } else if (NULL==(cfunc=H5Z_g[method].compress)) {
+ /* No compress function */
+ HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0,
+ "compression method is not supported");
+
+ } else if (0==(ret_value=(cfunc)(comp->flags, comp->cd_size,
+ comp->client_data, nbytes,
+ src, nbytes, dst))) {
+ /* Compress failed */
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "compression failed");
+
+ } else if (ret_value>=nbytes) {
+ /* Output is not smaller than input */
+#ifdef H5Z_DEBUG
+ H5Z_g[method].comp.over += 1;
+ over = 1;
+#endif
+ HGOTO_DONE (0);
+ }
+
+ done:
+#ifdef H5Z_DEBUG
+ H5Z_g[method].comp.nbytes += nbytes;
+ if (0==ret_value && !over) H5Z_g[method].comp.failed += nbytes;
+ H5_timer_end (&(H5Z_g[method].comp.timer), &timer);
+#endif
+ FUNC_LEAVE (ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_uncompress
+ *
+ * Purpose: Uncompress SRC_NBYTES from SRC into at most DST_NBYTES of
+ * DST.
+ *
+ * Return: Success: Number of bytes in DST buffer.
+ *
+ * Failure: 0 if the uncompression failed or DST wasn't
+ * big enough to hold the result.
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Z_uncompress (const H5O_compress_t *comp, size_t src_nbytes, const void *src,
+ size_t dst_nbytes, void *dst/*out*/)
+{
+ size_t ret_value = 0;
+ H5Z_func_t ufunc = NULL;
+ H5Z_method_t method = comp ? comp->method : H5Z_NONE;
+#ifdef H5Z_DEBUG
+ H5_timer_t timer;
+#endif
+
+ FUNC_ENTER (H5Z_uncompress, 0);
+
+#ifdef H5Z_DEBUG
+ H5_timer_begin (&timer);
+#endif
+
+ if (H5Z_NONE==method) {
+ /* No compression method */
+ assert (src_nbytes<=dst_nbytes);
+ HDmemcpy (dst, src, src_nbytes);
+ ret_value = src_nbytes;
+
+ } else if (src_nbytes==dst_nbytes) {
+ /* Data is not compressed */
+#ifdef H5Z_DEBUG
+ H5Z_g[method].uncomp.over += src_nbytes;
+#endif
+ HDmemcpy (dst, src, src_nbytes);
+ ret_value = src_nbytes;
+
+ } else if (NULL==(ufunc=H5Z_g[method].uncompress)) {
+ /* No uncompress function */
+ HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0,
+ "uncompression method is not supported");
+
+ } else if (0==(ret_value=(ufunc)(comp->flags, comp->cd_size,
+ comp->client_data, src_nbytes,
+ src, dst_nbytes, dst))) {
+ /* Uncompress failed */
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "uncompression failed");
+ }
+
+ done:
+#ifdef H5Z_DEBUG
+ H5Z_g[method].uncomp.nbytes += dst_nbytes;
+ if (0==ret_value) H5Z_g[method].uncomp.failed += dst_nbytes;
+ H5_timer_end (&(H5Z_g[method].uncomp.timer), &timer);
+#endif
+
+ FUNC_LEAVE (ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_register
+ *
+ * Purpose: Same as the public version except this one allows compression
+ * methods to be set for predefined method numbers <16.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Z_register (H5Z_method_t method, const char *name, H5Z_func_t cfunc,
+ H5Z_func_t ufunc)
+{
+ FUNC_ENTER (H5Z_register, FAIL);
+
+ assert (method>=0 && method<=H5Z_MAXVAL);
+ H5MM_xfree (H5Z_g[method].name);
+ H5Z_g[method].name = H5MM_xstrdup (name);
+ H5Z_g[method].compress = cfunc;
+ H5Z_g[method].uncompress = ufunc;
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_zlib_c
+ *
+ * Purpose: Compress SRC_NBYTES bytes from SRC into at most DST_NBYTES of
+ * DST using the compression level as specified in FLAGS.
+ *
+ * Return: Success: Number of bytes compressed into DST limited
+ * by DST_NBYTES.
+ *
+ * Failure: 0
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+H5Z_zlib_c (unsigned int flags, size_t __unused__ cd_size,
+ const void __unused__ *client_data, size_t src_nbytes,
+ const void *src, size_t dst_nbytes, void *dst/*out*/)
+{
+ size_t ret_value = 0;
+#ifdef HAVE_ZLIB_H
+ const Bytef *z_src = (const Bytef*)src;
+ Bytef *z_dst = (Bytef*)dst;
+ uLongf z_dst_nbytes = (uLongf)dst_nbytes;
+ uLong z_src_nbytes = (uLong)src_nbytes;
+ int level = flags % 10;
+ int status;
+#endif
+
+ FUNC_ENTER (H5Z_zlib_c, 0);
+
+#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H)
+ status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, level);
+ if (Z_BUF_ERROR==status) {
+ ret_value = dst_nbytes;
+ } else if (Z_MEM_ERROR==status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate memory error");
+ } else if (Z_OK!=status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate error");
+ } else {
+ ret_value = z_dst_nbytes;
+ }
+#else
+ HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0,
+ "hdf5 was not compiled with zlib-1.0.2 or better");
+#endif
+
+ done:
+ FUNC_LEAVE (ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Z_zlib_u
+ *
+ * Purpose: Uncompress SRC_NBYTES from SRC into at most DST_NBYTES of
+ * DST.
+ *
+ * Return: Success: Number of bytes returned in DST.
+ *
+ * Failure: 0
+ *
+ * Programmer: Robb Matzke
+ * Thursday, April 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+H5Z_zlib_u (unsigned int __unused__ flags, size_t __unused__ cd_size,
+ const void __unused__ *client_data, size_t src_nbytes,
+ const void *src, size_t dst_nbytes, void *dst/*out*/)
+{
+ size_t ret_value = 0;
+#ifdef HAVE_ZLIB_H
+ const Bytef *z_src = (const Bytef*)src;
+ Bytef *z_dst = (Bytef*)dst;
+ uLongf z_dst_nbytes = (uLongf)dst_nbytes;
+ uLong z_src_nbytes = (uLong)src_nbytes;
+ int status;
+#endif
+
+ FUNC_ENTER (H5Z_zlib_u, 0);
+
+#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H)
+ status = uncompress (z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+ if (Z_BUF_ERROR==status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0,
+ "deflate destination buffer was too small");
+ } else if (Z_MEM_ERROR==status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate memory error");
+ } else if (Z_DATA_ERROR==status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate corrupted data");
+ } else if (Z_OK!=status) {
+ HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate error");
+ }
+ ret_value = z_dst_nbytes;
+#else
+ HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0,
+ "hdf5 was not compiled with zlib-1.0.2 or better");
+#endif
+
+ done:
+ FUNC_LEAVE (ret_value);
+}
diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h
new file mode 100644
index 0000000..288ec07
--- /dev/null
+++ b/src/H5Zprivate.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 1998 NCSA
+ * All rights reserved.
+ *
+ * Programmer: Robb Matzke <matzke@llnl.gov>
+ * Thursday, April 16, 1998
+ */
+#ifndef _H5Zprivate_H
+#define _H5Zprivate_H
+
+#include <H5Zpublic.h>
+
+struct H5O_compress_t; /*forward decl*/
+
+herr_t H5Z_register (H5Z_method_t method, const char *name,
+ H5Z_func_t compress, H5Z_func_t uncompress);
+size_t H5Z_compress (const struct H5O_compress_t *compress, size_t nbytes,
+ const void *src, void *dst/*out*/);
+size_t H5Z_uncompress (const struct H5O_compress_t *compress,
+ size_t src_nbytes, const void *src, size_t dst_nbytes,
+ void *dst/*out*/);
+
+#endif
diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h
new file mode 100644
index 0000000..daebf89
--- /dev/null
+++ b/src/H5Zpublic.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 1998 NCSA
+ * All rights reserved.
+ *
+ * Programmer: Robb Matzke <matzke@llnl.gov>
+ * Thursday, April 16, 1998
+ */
+#ifndef _H5Zpublic_H
+#define _H5Zpublic_H
+
+/*
+ * Compression methods. Method zero means no compression. Methods 1 through
+ * 15 are defined by the library. Methods 16-255 are user-defined.
+ */
+typedef int H5Z_method_t;
+#define H5Z_NONE 0 /*no compression, must be zero */
+#define H5Z_DEFLATE 1 /*deflation like gzip */
+#define H5Z_RES_2 2 /*reserved for internal use */
+#define H5Z_RES_3 3 /*reserved for internal use */
+#define H5Z_RES_4 4 /*reserved for internal use */
+#define H5Z_RES_5 5 /*reserved for internal use */
+#define H5Z_RES_6 6 /*reserved for internal use */
+#define H5Z_RES_7 7 /*reserved for internal use */
+#define H5Z_RES_8 8 /*reserved for internal use */
+#define H5Z_RES_9 9 /*reserved for internal use */
+#define H5Z_RES_10 10 /*reserved for internal use */
+#define H5Z_RES_11 11 /*reserved for internal use */
+#define H5Z_RES_12 12 /*reserved for internal use */
+#define H5Z_RES_13 13 /*reserved for internal use */
+#define H5Z_RES_14 14 /*reserved for internal use */
+#define H5Z_RES_15 15 /*reserved for internal use */
+/* user-defined 16-255 */
+#define H5Z_MAXVAL 255 /*maximum compression method ID */
+
+/*
+ * A compression function takes some configuration data which comes from the
+ * compression message, namely FLAGS, CD_SIZE, and CLIENT_DATA. It should
+ * read SRC_NBYTES from SRC and compress them into at most DST_NBYTES of DST.
+ * If the compressed data would be larger than DST_NBYTES the function should
+ * return a value greater than or equal to DST_NBYTES. On failure the
+ * function may return zero.
+ *
+ * The uncompression function is the inverse of compression and takes the
+ * same arguments. The SRC_NBYTES argument is the number of compressed bytes
+ * in SRC. The function should uncompress SRC into DST. For redundancy,
+ * DST_NBYTES contains the size of the DST buffer although if the algorithm
+ * is operating properly and the file has not been corrupted the uncompressed
+ * data will never be larger than DST_NBYTES. The function should return the
+ * number of bytes in the DST buffer or zero on failure. Failure includes
+ * the overflow of the DST buffer.
+ */
+typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_size,
+ const void *client_data, size_t src_nbytes,
+ const void *src, size_t dst_nbytes,
+ void *dst/*out*/);
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+herr_t H5Zregister (H5Z_method_t method, const char *name, H5Z_func_t compress,
+ H5Z_func_t uncompress);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+
diff --git a/src/H5config.h.in b/src/H5config.h.in
index a715025..9bb277a 100644
--- a/src/H5config.h.in
+++ b/src/H5config.h.in
@@ -64,6 +64,9 @@
/* Define if you have the getpwuid function. */
#undef HAVE_GETPWUID
+/* Define if you have the getrusage function. */
+#undef HAVE_GETRUSAGE
+
/* Define if you have the lseek64 function. */
#undef HAVE_LSEEK64
@@ -73,6 +76,9 @@
/* Define if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
+/* Define if you have the <zlib.h> header file. */
+#undef HAVE_ZLIB_H
+
/* Define if you have the coug library (-lcoug). */
#undef HAVE_LIBCOUG
@@ -81,3 +87,6 @@
/* Define if you have the mpio library (-lmpio). */
#undef HAVE_LIBMPIO
+
+/* Define if you have the z library (-lz). */
+#undef HAVE_LIBZ
diff --git a/src/H5private.h b/src/H5private.h
index 33493da..328f907 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -206,6 +206,19 @@ typedef struct {
#endif
/*
+ * Data types and functions for timing certain parts of the library.
+ */
+typedef struct {
+ double utime; /*user time */
+ double stime; /*system time */
+ double etime; /*elapsed wall-clock time */
+} H5_timer_t;
+
+void H5_timer_reset (H5_timer_t *timer);
+void H5_timer_begin (H5_timer_t *timer);
+void H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/);
+
+/*
* Redefine all the POSIX functions. We should never see a POSIX
* function (or any other non-HDF5 function) in the source!
*/
diff --git a/src/Makefile.in b/src/Makefile.in
index 20b4ae3..ca59511 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -18,10 +18,10 @@ PARALLEL_SRC=H5Fmpio.c
LIB_SRC=H5.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcore.c H5Ffamily.c \
H5Fistore.c H5Flow.c H5Fsec2.c H5Fsplit.c H5Fstdio.c H5G.c H5Gent.c \
- H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c H5Ocont.c \
- H5Odtype.c H5Oefl.c H5Olayout.c H5Oname.c H5Onull.c H5Osdspace.c \
- H5Oshared.c H5Ostab.c H5P.c H5S.c H5Ssimp.c H5T.c H5Tconv.c H5Tinit.c \
- H5V.c @PARALLEL_SRC@
+ H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c H5Ocomp.c \
+ H5Ocont.c H5Odtype.c H5Oefl.c H5Olayout.c H5Oname.c H5Onull.c \
+ H5Osdspace.c H5Oshared.c H5Ostab.c H5P.c H5S.c H5Ssimp.c H5T.c \
+ H5Tconv.c H5Tinit.c H5V.c H5Z.c @PARALLEL_SRC@
LIB_OBJ=$(LIB_SRC:.c=.o)
@@ -36,13 +36,13 @@ PROG_OBJ=$(PROG_SRC:.c=.o)
PUB_HDR=H5public.h H5ACpublic.h H5Bpublic.h H5Ppublic.h H5Dpublic.h \
H5Epublic.h H5Fpublic.h H5Gpublic.h H5HGpublic.h H5HLpublic.h \
H5Ipublic.h H5MFpublic.h H5MMpublic.h H5Opublic.h H5Spublic.h \
- H5Tpublic.h H5config.h hdf5.h
+ H5Tpublic.h H5Zpublic.h H5config.h hdf5.h
# Other header files (not to be installed)...
PRIVATE_HDR=H5private.h H5ACprivate.h H5Bprivate.h H5Pprivate.h H5Dprivate.h \
H5Eprivate.h H5Fprivate.h H5Gprivate.h H5Gpkg.h H5HGprivate.h \
H5HLprivate.h H5Iprivate.h H5MFprivate.h H5MMprivate.h H5Oprivate.h \
- H5Sprivate.h H5Tprivate.h H5Tpkg.h H5Vprivate.h
+ H5Sprivate.h H5Tprivate.h H5Tpkg.h H5Vprivate.h H5Zprivate.h
# Number format detection
H5Tinit.c: H5detect
diff --git a/src/debug.c b/src/debug.c
index f5370b4..4b8fdc3 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -26,6 +26,7 @@
#define INDENT 3
#define VCOL 50
+
/*-------------------------------------------------------------------------
* Function: main
@@ -51,7 +52,7 @@ main(int argc, char *argv[])
H5F_t *f;
haddr_t addr;
uint8 sig[16];
- intn i;
+ intn i, ndims;
herr_t status = SUCCEED;
haddr_t extra;
@@ -123,12 +124,18 @@ main(int argc, char *argv[])
* subclass. The subclass identifier is the byte immediately
* after the B-tree signature.
*/
- H5B_subid_t subtype = (H5B_subid_t)sig[H5B_SIZEOF_MAGIC];
+ H5B_subid_t subtype = (H5B_subid_t)sig[H5B_SIZEOF_MAGIC];
+
switch (subtype) {
case H5B_SNODE_ID:
status = H5G_node_debug(f, &addr, stdout, 0, VCOL, &extra);
break;
+ case H5B_ISTORE_ID:
+ ndims = (int)extra.offset;
+ status = H5F_istore_debug (f, &addr, stdout, 0, VCOL, ndims);
+ break;
+
default:
fprintf(stderr, "Unknown B-tree subtype %u\n", (unsigned) (subtype));
HDexit(4);
diff --git a/src/h5ls.c b/src/h5ls.c
index 23ef00c..ca2c139 100644
--- a/src/h5ls.c
+++ b/src/h5ls.c
@@ -44,7 +44,7 @@ list (hid_t group, const char *name, void __unused__ *op_data)
hid_t (*func)(void*);
void *edata;
int i;
- char linkval[512];
+ char buf[512];
H5G_stat_t statbuf;
/* Disable error reporting */
@@ -52,19 +52,20 @@ list (hid_t group, const char *name, void __unused__ *op_data)
H5Eset_auto (NULL, NULL);
/* Print info about each name */
- printf ("%-30s", name);
+ printf ("%-25s ", name);
if (H5Gstat (group, name, TRUE, &statbuf)>=0) {
- printf ("FILE={%lu,%lu}, OID={%lu,%lu} ",
- statbuf.fileno[0], statbuf.fileno[1],
- statbuf.objno[0], statbuf.objno[1]);
+ sprintf (buf, "%lu:%lu:%lu:%lu",
+ statbuf.fileno[1], statbuf.fileno[0],
+ statbuf.objno[1], statbuf.objno[0]);
+ printf ("%-20s ", buf);
}
if ((obj=H5Dopen (group, name))>=0) {
hsize_t size[64];
hid_t space = H5Dget_space (obj);
int ndims = H5Sget_dims (space, size);
- printf (" Dataset {");
+ printf ("Dataset {");
for (i=0; i<ndims; i++) {
HDfprintf (stdout, "%s%Hu", i?", ":"", size[i]);
}
@@ -72,15 +73,15 @@ list (hid_t group, const char *name, void __unused__ *op_data)
H5Dclose (space);
H5Dclose (obj);
} else if ((obj=H5Gopen (group, name))>=0) {
- printf (" Group\n");
+ printf ("Group\n");
H5Gclose (obj);
- } else if (H5Gget_linkval (group, name, sizeof(linkval), linkval)>=0) {
- if (NULL==HDmemchr (linkval, 0, sizeof(linkval))) {
- strcpy (linkval+sizeof(linkval)-4, "...");
+ } else if (H5Gget_linkval (group, name, sizeof(buf), buf)>=0) {
+ if (NULL==HDmemchr (buf, 0, sizeof(buf))) {
+ strcpy (buf+sizeof(buf)-4, "...");
}
- printf (" -> %s\n", linkval);
+ printf (" -> %s\n", buf);
} else {
- printf (" Unknown Type\n");
+ printf ("Unknown Type\n");
}
/* Restore error reporting */
diff --git a/src/hdf5.h b/src/hdf5.h
index 8c71ace..ce4ed62 100644
--- a/src/hdf5.h
+++ b/src/hdf5.h
@@ -34,4 +34,6 @@
#include <H5Ppublic.h> /* Property lists */
#include <H5Spublic.h> /* Dataspaces */
#include <H5Tpublic.h> /* Datatypes */
+#include <H5Zpublic.h> /* Data compression */
+
#endif