diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | src/H5.c | 12 | ||||
| -rw-r--r-- | src/H5AC.c | 19 | ||||
| -rw-r--r-- | src/H5D.c | 8 | ||||
| -rw-r--r-- | src/H5Distore.c | 934 | ||||
| -rw-r--r-- | src/H5F.c | 83 | ||||
| -rw-r--r-- | src/H5Fistore.c | 934 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 20 | ||||
| -rw-r--r-- | src/H5MM.c | 130 | ||||
| -rw-r--r-- | src/H5O.c | 77 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 2 | ||||
| -rw-r--r-- | src/H5P.c | 292 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 4 | ||||
| -rw-r--r-- | src/H5Z.c | 16 | ||||
| -rw-r--r-- | src/H5Zpublic.h | 35 | ||||
| -rw-r--r-- | src/Makefile.in | 8 | ||||
| -rw-r--r-- | test/.distdep | 22 | ||||
| -rw-r--r-- | test/Makefile.in | 16 | ||||
| -rw-r--r-- | test/big.c | 4 | ||||
| -rw-r--r-- | test/chunk.c | 521 | ||||
| -rw-r--r-- | test/cmpd_dset.c | 2 | ||||
| -rw-r--r-- | test/extend.c | 3 | ||||
| -rw-r--r-- | test/external.c | 9 | ||||
| -rw-r--r-- | test/gheap.c | 20 |
24 files changed, 2520 insertions, 652 deletions
@@ -206,6 +206,7 @@ ./test/.distdep ./test/Makefile.in ./test/big.c +./test/chunk.c ./test/cmpd_dset.c ./test/dsets.c ./test/dtypes.c @@ -535,7 +535,7 @@ HDfprintf (FILE *stream, const char *fmt, ...) } /* Type modifier */ - if (strchr ("Hhlq", *s)) { + if (strchr ("ZHhlq", *s)) { switch (*s) { case 'H': if (sizeof(hsize_t)==sizeof(long)) { @@ -544,6 +544,16 @@ HDfprintf (FILE *stream, const char *fmt, ...) strcpy (modifier, PRINTF_LL_WIDTH); } break; + case 'Z': + if (sizeof(size_t)==sizeof(long)) { + strcpy (modifier, "l"); + } else if (sizeof(size_t)==sizeof(long long)) { + strcpy (modifier, PRINTF_LL_WIDTH); + } else if (sizeof(size_t)==sizeof(int)) { + modifier[0] = '\0'; + } + break; + default: modifier[0] = *s; modifier[1] = '\0'; @@ -50,7 +50,7 @@ static H5AC_t *current_cache_g = NULL; /*for sorting */ * pass an invalid value then H5AC_NSLOTS is used. You can * turn off caching by using 1 for the SIZE_HINT value. * - * Return: Success: SUCCEED + * Return: Success: Number of slots actually used. * * Failure: FAIL * @@ -62,7 +62,7 @@ static H5AC_t *current_cache_g = NULL; /*for sorting */ * *------------------------------------------------------------------------- */ -herr_t +intn H5AC_create(H5F_t *f, intn size_hint) { H5AC_t *cache = NULL; @@ -70,14 +70,13 @@ H5AC_create(H5F_t *f, intn size_hint) assert(f); assert(NULL == f->shared->cache); - if (size_hint < 1) - size_hint = H5AC_NSLOTS; + if (size_hint < 1) size_hint = H5AC_NSLOTS; f->shared->cache = cache = H5MM_xcalloc(1, sizeof(H5AC_t)); cache->nslots = size_hint; cache->slot = H5MM_xcalloc((intn)(cache->nslots), sizeof(H5AC_slot_t)); - FUNC_LEAVE(SUCCEED); + FUNC_LEAVE(size_hint); } /*------------------------------------------------------------------------- @@ -838,7 +837,7 @@ H5AC_debug(H5F_t *f) FUNC_ENTER(H5AC_debug, FAIL); - fprintf(stderr, "H5AC: cache statistics for file %s\n", f->name); + fprintf(stderr, "H5AC: meta data cache statistics for file %s\n", f->name); fprintf(stderr, " %-18s %8s %8s %8s %8s+%-8s\n", "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes"); fprintf(stderr, " %-18s %8s %8s %8s %8s-%-8s\n", @@ -866,10 +865,12 @@ H5AC_debug(H5F_t *f) sprintf(s, "unknown id %d", i); } - if (cache->diagnostics[i].nhits) { + if (cache->diagnostics[i].nhits>0 || + cache->diagnostics[i].nmisses>0) { miss_rate = 100.0 * cache->diagnostics[i].nmisses / - cache->diagnostics[i].nhits; - } else { + (cache->diagnostics[i].nhits+ + cache->diagnostics[i].nmisses); + } else { miss_rate = 0.0; } @@ -1252,8 +1252,8 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, mem_space, buf/*out*/); if (status>=0) goto succeed; #ifdef H5D_DEBUG - fprintf (stderr, "H5D: input pipe optimization failed " - "(falling through)\n"); + fprintf (stderr, "H5D: data space conversion could not be optimized " + "for this case (using general method instead)\n"); #endif H5E_clear (); } @@ -1547,8 +1547,8 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, mem_space, buf); if (status>=0) goto succeed; #ifdef H5D_DEBUG - fprintf (stderr, "H5D: output pipe optimization failed " - "(falling through)\n"); + fprintf (stderr, "H5D: data space conversion could not be optimized " + "for this case (using general method instead)\n"); #endif H5E_clear (); } diff --git a/src/H5Distore.c b/src/H5Distore.c index 7e80f1a..91c4e9d 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -2,8 +2,17 @@ * Copyright (C) 1997 NCSA * All rights reserved. * - * Programmer: Robb Matzke <matzke@llnl.gov> - * Wednesday, October 8, 1997 + * Programmer: Robb Matzke <matzke@llnl.gov> + * Wednesday, October 8, 1997 + * + * Purpose: Indexed (chunked) I/O functions. The logical + * multi-dimensional data space is regularly partitioned into + * same-sized "chunks", the first of which is aligned with the + * logical origin. The chunks are given a multi-dimensional + * index which is used as a lookup key in a B-tree that maps + * chunk index to disk address. Each chunk can be compressed + * independently and the chunks may move around in the file as + * their storage requirements change. */ #include <H5private.h> #include <H5Dprivate.h> @@ -14,13 +23,25 @@ #include <H5Oprivate.h> #include <H5Vprivate.h> - /* Interface initialization */ #define PABLO_MASK H5F_istore_mask static hbool_t interface_initialize_g = FALSE; #define INTERFACE_INIT NULL -/* PRIVATE PROTOTYPES */ +/* Raw data chunks are cached. Each entry in the cache is: */ +typedef struct H5F_rdcc_ent_t { + hbool_t locked; /*entry is locked in cache */ + hbool_t dirty; /*needs to be written to disk? */ + H5O_layout_t *layout; /*the layout message */ + H5O_compress_t *comp; /*compression message */ + hssize_t offset[H5O_LAYOUT_NDIMS]; /*chunk name */ + size_t chunk_size; /*size of a chunk */ + size_t rd_count; /*bytes remaining to be read */ + size_t wr_count; /*bytes remaining to be written */ + uint8 *chunk; /*the uncompressed chunk data */ +} H5F_rdcc_ent_t; + +/* Private prototypes */ static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata); static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key, void *_udata, void *_rt_key, haddr_t *); @@ -546,7 +567,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, * Already exists. If the new size is not the same as the old size * then we should reallocate storage. */ -#if 1 if (lt_key->nbytes != udata->key.nbytes) { if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr, udata->key.nbytes, new_node/*out*/)<0) { @@ -561,13 +581,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, udata->addr = *addr; ret_value = H5B_INS_NOOP; } -#else - assert (lt_key->nbytes == udata->key.nbytes); - assert (!H5F_addr_defined (&(udata->addr)) || - H5F_addr_eq (&(udata->addr), addr)); - udata->addr = *addr; - ret_value = H5B_INS_NOOP; -#endif } else if (H5V_hyper_disjointp(udata->mesg.ndims, lt_key->offset, udata->mesg.dim, @@ -606,6 +619,596 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- + * Function: H5F_istore_init + * + * Purpose: Initialize the raw data chunk cache for a file. This is + * called when the file handle is initialized. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Monday, May 18, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_init (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + + FUNC_ENTER (H5F_istore_init, FAIL); + + HDmemset (rdcc, 0, sizeof(H5F_rdcc_t)); + if (f->shared->access_parms->rdcc_nbytes>0) { + rdcc->nslots = 25; /*some initial number of slots*/ + rdcc->slot = H5MM_xcalloc (rdcc->nslots, sizeof(H5F_rdcc_ent_t)); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_flush_entry + * + * Purpose: Writes a chunk to disk. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_flush_entry (H5F_t *f, H5F_rdcc_ent_t *ent) +{ + void *c_buf = NULL; /*temp compression buffer */ + void *out_ptr = NULL; /*ptr to output buffer */ + size_t nbytes; /*size of output buffer */ + herr_t ret_value = FAIL; /*return value */ + H5F_istore_ud1_t udata; /*pass through B-tree */ + intn i; + + FUNC_ENTER (H5F_istore_flush_entry, FAIL); + assert (ent); + assert (!ent->locked); + if (!ent->dirty) HRETURN (SUCCEED); + + /* Should the chunk be compressed before writing it to disk? */ + if (ent->comp && H5Z_NONE!=ent->comp->method) { + c_buf = H5MM_xmalloc (ent->chunk_size); + nbytes = H5Z_compress (ent->comp, ent->chunk_size, ent->chunk, c_buf); + if (nbytes && nbytes<ent->chunk_size) { + out_ptr = c_buf; + } else { + out_ptr = ent->chunk; + nbytes = ent->chunk_size; + } + } else { + out_ptr = ent->chunk; + nbytes = ent->chunk_size; + } + + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk if its + * size changed. Then write the data into the file. + */ + udata.mesg = *(ent->layout); + H5F_addr_undef(&(udata.addr)); + udata.key.nbytes = nbytes; + for (i=0; i<ent->layout->ndims; i++) { + udata.key.offset[i] = ent->offset[i]; + } + + if (H5B_insert(f, H5B_ISTORE, &(ent->layout->addr), &udata)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to allocate chunk"); + } + if (H5F_block_write (f, &(udata.addr), nbytes, out_ptr)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to write raw data to file"); + } + + /* Mark cache entry as clean */ + ent->dirty = FALSE; + f->shared->rdcc.nflushes++; + ret_value = SUCCEED; + + done: + H5MM_xfree (c_buf); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_flush + * + * Purpose: Writes all dirty chunks to disk but does not remove them from + * the cache. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_flush (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + intn i, nerrors=0; + + FUNC_ENTER (H5F_istore_flush, FAIL); + + for (i=0; i<rdcc->nused; i++) { + if (H5F_istore_flush_entry (f, rdcc->slot+i)<0) { + nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to flush one or more raw data chunks"); + } + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_preempt + * + * Purpose: Preempts the specified entry from the cache, flushing it to + * disk if necessary. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_preempt (H5F_t *f, intn idx) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = rdcc->slot + idx; + + FUNC_ENTER (H5F_istore_preempt, FAIL); + assert (idx>=0 && idx<rdcc->nused); + assert (!ent->locked); + + if (ent->dirty) H5F_istore_flush_entry (f, ent); + H5O_free (H5O_LAYOUT, ent->layout); + H5O_free (H5O_COMPRESS, ent->comp); + H5MM_xfree (ent->chunk); + rdcc->nused -= 1; + rdcc->nbytes -= ent->chunk_size; + HDmemmove (rdcc->slot+idx, rdcc->slot+idx+1, + (rdcc->nused-idx) * sizeof(H5F_rdcc_ent_t)); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_dest + * + * Purpose: Destroy the entire chunk cache by flushing dirty entries, + * preempting all entries, and freeing the cache itself. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_dest (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + intn i, nerrors=0; + + FUNC_ENTER (H5F_istore_dest, FAIL); + + for (i=rdcc->nused-1; i>=0; --i) { + if (H5F_istore_flush_entry (f, rdcc->slot+i)<0) { + nerrors++; + } + if (H5F_istore_preempt (f, i)<0) { + nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to flush one or more raw data chunks"); + } + + H5MM_xfree (rdcc->slot); + HDmemset (rdcc, 0, sizeof(H5F_rdcc_t)); + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune + * + * Purpose: Prune the cache by preempting some things until the cache has + * room for something which is SIZE bytes. Only unlocked + * entries are considered for preemption. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_prune (H5F_t *f, size_t size) +{ + intn i, meth0, meth1, nerrors=0; + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent0, *ent1; + double w0 = f->shared->access_parms->rdcc_w0; + size_t total = f->shared->access_parms->rdcc_nbytes; + + FUNC_ENTER (H5F_istore_prune, FAIL); + + /* + * We have two pointers that slide down the cache beginning at the least + * recently used entry. The distance between the pointers represents the + * relative weight. A weight of 50% for the first pointer means that the + * second pointer is half the cache length behind the first pointer. + */ + meth0 = rdcc->nused; + meth1 = rdcc->nused * (1.0+w0); + for (i=MAX(meth0, meth1)-1; + rdcc->nbytes+size>total && i>=0; + --i, --meth0, --meth1) { + + ent0 = rdcc->slot+meth0; /*might be a bad pointer!*/ + ent1 = rdcc->slot+meth1; /*might be a bad pointer!*/ + + if (meth0>=0 && meth0<rdcc->nused && !ent0->locked && + (0==ent0->rd_count || ent0->chunk_size==ent0->rd_count) && + (0==ent0->wr_count || ent0->chunk_size==ent0->wr_count)) { + /* + * Method 0: Preempt entries that have a zero rd_count. If the + * application is accessing a dataset with a set of + * non-overlapping partial I/O requests then chunks with a zero + * rd_count will probably not be accessed in the near future. + */ + if (H5F_istore_preempt (f, meth0)<0) nerrors++; + + } else if (meth1>=0 && meth1<rdcc->nused && !ent1->locked) { + /* + * Method 1: Discard the least recently used members from the + * cache. This is a catch-all. + */ + if (H5F_istore_preempt (f, meth1)<0) nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to preempt one or more raw data cache entry"); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_lock + * + * Purpose: Return a pointer to an uncompressed chunk. The pointer + * points directly into the chunk cache and should not be freed + * by the caller but will be valid until it is unlocked. The + * input value IDX_HINT is used to speed up cache lookups and + * it's output value should be given to H5F_rdcc_unlock(). + * + * If RELAX is non-zero and the chunk isn't in the cache then + * don't try to read it from the file, but just allocate an + * uninitialized buffer to hold the result. This is indented + * for output functions that are about to overwrite the entire + * chunk. + * + * Return: Success: Ptr to an uncompressed chunk. + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5F_istore_lock (H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, const hssize_t offset[], + hbool_t relax, intn *idx_hint/*in,out*/) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = NULL; + intn i, j, found = -1; + H5F_istore_ud1_t udata; /*B-tree pass-through */ + size_t chunk_size; /*size of a chunk */ + herr_t status; /*func return status */ + void *chunk=NULL; /*the uncompressed chunk*/ + void *temp=NULL; /*temporary chunk buffer*/ + void *ret_value=NULL; /*return value */ + + FUNC_ENTER (H5F_istore_lock, NULL); + + /* First use the hint */ + if (idx_hint && *idx_hint>=0 && *idx_hint<rdcc->nused) { + ent = rdcc->slot + *idx_hint; + if (layout->ndims==ent->layout->ndims) { + for (i=0, found=*idx_hint; found>=0 && i<ent->layout->ndims; i++) { + if (offset[i]!=ent->offset[i]) found = -1; + } + } + } + + /* Then look at all the entries */ + for (i=0; found<0 && i<rdcc->nused; i++) { + ent = rdcc->slot + i; + if (layout->ndims==ent->layout->ndims) { + for (j=0, found=i; found>=0 && j<ent->layout->ndims; j++) { + if (offset[j]!=ent->offset[j]) found = -1; + } + } + } + + + if (found>=0) { + /* + * Already in the cache. Count a hit. + */ + rdcc->nhits++; + + } else if (found<0 && relax) { + /* + * Not in the cache, but we're about to overwrite the whole thing + * anyway, so just allocate a buffer for it but don't initialize that + * buffer with the file contents. Count this as a hit instead of a + * miss because we saved ourselves lots of work. + */ + rdcc->nhits++; + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc (chunk_size); + + } else { + /* + * Not in the cache. Read it from the file and count this as a miss + * if it's in the file or an init if it isn't. + */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + udata.key.offset[i] = offset[i]; + chunk_size *= layout->dim[i]; + } + udata.mesg = *layout; + H5F_addr_undef (&(udata.addr)); + status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); + chunk = H5MM_xmalloc (chunk_size); + if (status>=0 && H5F_addr_defined (&(udata.addr))) { + /* + * The chunk exists on disk but might be compressed. Instead of + * allocating the exact size for the compressed chunk we allocate + * the entire chunk size -- it reduces strain on the malloc() + * subsystem. + */ + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, chunk)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, + "unable to read raw data chunk"); + } + if (udata.key.nbytes<chunk_size) { + temp = H5MM_xmalloc (chunk_size); + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + chunk, chunk_size, temp)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, + "unable to uncompress raw data chunk"); + } + H5MM_xfree (chunk); + chunk = temp; + temp = NULL; + } + rdcc->nmisses++; + } else { + /* + * The chunk doesn't exist in the file. Assume all zeros. + */ + HDmemset (chunk, 0, chunk_size); + rdcc->ninits++; + } + } + + if (found<0 && chunk_size<=f->shared->access_parms->rdcc_nbytes) { + /* + * Add the chunk to the beginning of the cache after pruning the cache + * to make room. + */ + if (H5F_istore_prune (f, chunk_size)<0) { + H5E_clear (); + } + if (rdcc->nused>=rdcc->nslots) { + rdcc->nslots = MAX (25, 2*rdcc->nslots); + rdcc->slot = H5MM_xrealloc (rdcc->slot, + (rdcc->nslots* + sizeof(H5F_rdcc_ent_t))); + } + HDmemmove (rdcc->slot+1, rdcc->slot, + rdcc->nused*sizeof(H5F_rdcc_ent_t)); + rdcc->nused++; + rdcc->nbytes += chunk_size; + ent = rdcc->slot; + ent->locked = 0; + ent->dirty = FALSE; + ent->chunk_size = chunk_size; + ent->layout = H5O_copy (H5O_LAYOUT, layout); + ent->comp = H5O_copy (H5O_COMPRESS, comp); + for (i=0; i<layout->ndims; i++) { + ent->offset[i] = offset[i]; + } + ent->rd_count = chunk_size; + ent->wr_count = chunk_size; + ent->chunk = chunk; + found = 0; + + } else if (found<0) { + /* + * The chunk is larger than the entire cache so we don't cache it. + * This is the reason all those arguments have to be repeated for the + * unlock function. + */ + ent = NULL; + found = -999; + + } else if (found>0) { + /* + * The chunk is not at the beginning of the cache; move it forward by + * one slot. This is how we implement the LRU preemption algorithm. + */ + H5F_rdcc_ent_t x = rdcc->slot[found]; + rdcc->slot[found] = rdcc->slot[found-1]; + rdcc->slot[found-1] = x; + ent = rdcc->slot + --found; + } + + /* Lock the chunk into the cache */ + if (ent) { + assert (!ent->locked); + ent->locked = TRUE; + if (idx_hint) *idx_hint = found; + chunk = ent->chunk; + } + + ret_value = chunk; + done: + if (!ret_value) H5MM_xfree (chunk); + H5MM_xfree (temp); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_unlock + * + * Purpose: Unlocks a previously locked chunk. The LAYOUT, COMP, and + * OFFSET arguments should be the same as for H5F_rdcc_lock(). + * The DIRTY argument should be set to non-zero if the chunk has + * been modified since it was locked. The IDX_HINT argument is + * the returned index hint from the lock operation and BUF is + * the return value from the lock. + * + * The NACCESSED argument should be the number of bytes accessed + * for reading or writing (depending on the value of DIRTY). + * It's only purpose is to provide additional information to the + * preemption policy. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_unlock (H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, hbool_t dirty, + const hssize_t offset[], intn *idx_hint, + uint8 *chunk, size_t naccessed) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = NULL; + intn i, found = -1; + + FUNC_ENTER (H5F_istore_unlock, FAIL); + + /* First look at the hint */ + if (idx_hint && *idx_hint>=0 && *idx_hint<rdcc->nused) { + if (rdcc->slot[*idx_hint].chunk==chunk) found = *idx_hint; + } + + /* Then look at all the entries */ + for (i=0; found<0 && i<rdcc->nused; i++) { + if (rdcc->slot[i].chunk==chunk) found = i; + } + + if (found<0) { + /* + * It's not in the cache, probably because it's too big. If it's + * dirty then flush it to disk. In any case, free the chunk. + * Note: we have to copy the layout and compression messages so we + * don't discard the `const' qualifier. + */ + if (dirty) { + H5F_rdcc_ent_t x; + HDmemset (&x, 0, sizeof x); + x.dirty = TRUE; + x.layout = H5O_copy (H5O_LAYOUT, layout); + x.comp = H5O_copy (H5O_COMPRESS, comp); + for (i=0; i<layout->ndims; i++) { + x.offset[i] = offset[i]; + } + x.chunk = chunk; + H5F_istore_flush_entry (f, &x); + H5O_free (H5O_LAYOUT, x.layout); + H5O_free (H5O_COMPRESS, x.comp); + } + H5MM_xfree (chunk); + } else { + /* + * It's in the cache so unlock it. + */ + ent = rdcc->slot + found; + assert (ent->locked); + if (dirty) { + ent->dirty = TRUE; + ent->wr_count -= MIN (ent->wr_count, naccessed); + } else { + ent->rd_count -= MIN (ent->rd_count, naccessed); + } + ent->locked = FALSE; + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_read * * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw @@ -629,18 +1232,17 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, { hssize_t offset_m[H5O_LAYOUT_NDIMS]; hsize_t size_m[H5O_LAYOUT_NDIMS]; - intn i, carry; hsize_t idx_cur[H5O_LAYOUT_NDIMS]; hsize_t idx_min[H5O_LAYOUT_NDIMS]; hsize_t idx_max[H5O_LAYOUT_NDIMS]; hsize_t sub_size[H5O_LAYOUT_NDIMS]; hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; - size_t chunk_size; - uint8 *chunk=NULL, *compressed=NULL; - H5F_istore_ud1_t udata; - herr_t status; - herr_t ret_value = FAIL; + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; + intn i, carry; + size_t naccessed; /*bytes accessed in chnk*/ + uint8 *chunk=NULL; /*ptr to a chunk buffer */ + intn idx_hint=0; /*cache index hint */ FUNC_ENTER(H5F_istore_read, FAIL); @@ -672,61 +1274,10 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, } #endif - /* Determine the chunk size and allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - if (comp && H5Z_NONE!=comp->method) { - compressed = H5MM_xmalloc (chunk_size); - } - - /* - * As a special case if the source is aligned on a chunk boundary and is - * the same size as a chunk, and the destination is the same size as a - * chunk, then instead of reading into a temporary buffer and then into - * the destination, we read directly into the destination. - */ - for (i=0; i<layout->ndims; i++) { - if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ - if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ - if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ - udata.key.offset[i] = offset_f[i]; - } - if (i==layout->ndims) { - udata.mesg = *layout; - H5F_addr_undef (&(udata.addr)); - status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined (&(udata.addr))) { - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, buf)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress raw storage chunk"); - } - } else { - assert (udata.key.nbytes==chunk_size); - if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset (buf, 0, chunk_size); - } - HGOTO_DONE (SUCCEED); - } - /* - * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing - * or copying each chunk into a temporary buffer, and then copying it to - * it's destination. + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each to its final destination in the + * application buffer. */ for (i=0; i<layout->ndims; i++) { idx_min[i] = offset_f[i] / layout->dim[i]; @@ -734,59 +1285,42 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, idx_cur[i] = idx_min[i]; } - /* Initialize non-changing part of udata */ - udata.mesg = *layout; - /* Loop over all chunks */ while (1) { - - for (i=0; i<layout->ndims; i++) { + for (i=0, naccessed=1; i<layout->ndims; i++) { /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); - udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + chunk_offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - - udata.key.offset[i]; + offset_wrt_chunk[i] = MAX(offset_f[i], chunk_offset[i]) - + chunk_offset[i]; sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], offset_f[i]+size[i]) - - (udata.key.offset[i] + offset_wrt_chunk[i]); + (chunk_offset[i] + offset_wrt_chunk[i]); + naccessed *= sub_size[i]; /* Offset into mem buffer */ - sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + sub_offset_m[i] = chunk_offset[i] + offset_wrt_chunk[i] + offset_m[i] - offset_f[i]; } - /* Read chunk */ - H5F_addr_undef(&(udata.addr)); - status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined(&(udata.addr))) { - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, - chunk)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress data"); - } - } else { - assert (udata.key.nbytes == chunk_size); - if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset(chunk, 0, chunk_size); + /* + * Lock the chunk, transfer data to the application, then unlock the + * chunk. + */ + if (NULL==(chunk=H5F_istore_lock (f, layout, comp, chunk_offset, + FALSE, &idx_hint))) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw data chunk"); } - - /* Transfer data from the chunk buffer to the application */ H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m, - (void *)buf, layout->dim, offset_wrt_chunk, chunk); + (void*)buf, layout->dim, offset_wrt_chunk, chunk); + if (H5F_istore_unlock (f, layout, comp, FALSE, chunk_offset, &idx_hint, + chunk, naccessed)<0) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to unlock raw data chunk"); + } /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { @@ -795,12 +1329,7 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, } if (carry) break; } - ret_value = SUCCEED; - - done: - H5MM_xfree(chunk); - H5MM_xfree (compressed); - FUNC_LEAVE(ret_value); + FUNC_LEAVE(SUCCEED); } @@ -833,12 +1362,12 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, hsize_t idx_min[H5O_LAYOUT_NDIMS]; hsize_t idx_max[H5O_LAYOUT_NDIMS]; hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; - hsize_t chunk_size, nbytes; - uint8 *chunk=NULL, *compressed=NULL, *outbuf; - H5F_istore_ud1_t udata; - herr_t ret_value = FAIL; + uint8 *chunk=NULL; + intn idx_hint=0; + size_t chunk_size, naccessed; FUNC_ENTER(H5F_istore_write, FAIL); @@ -855,9 +1384,10 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, * For now the source must not be a hyperslab. It must be an entire * memory buffer. */ - for (i=0; i<layout->ndims; i++) { + for (i=0, chunk_size=1; i<layout->ndims; i++) { offset_m[i] = 0; size_m[i] = size[i]; + chunk_size *= layout->dim[i]; } #ifndef NDEBUG @@ -871,10 +1401,9 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, #endif /* - * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks copying - * each chunk into a temporary buffer, compressing or decompressing, and - * then copying it to it's destination. + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each chunk from the application to the + * chunk cache. */ for (i=0; i<layout->ndims; i++) { idx_min[i] = offset_f[i] / layout->dim[i]; @@ -882,102 +1411,46 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, idx_cur[i] = idx_min[i]; } - /* Allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - if (comp && H5Z_NONE!=comp->method) { - compressed = H5MM_xmalloc (chunk_size); - } - - /* Initialize non-changing part of udata */ - udata.mesg = *layout; /* Loop over all chunks */ while (1) { - - for (i=0; i<layout->ndims; i++) { + + for (i=0, naccessed=1; i<layout->ndims; i++) { /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); - udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + chunk_offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - - udata.key.offset[i]; + offset_wrt_chunk[i] = MAX(offset_f[i], chunk_offset[i]) - + chunk_offset[i]; sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], offset_f[i]+size[i]) - - (udata.key.offset[i] + offset_wrt_chunk[i]); + (chunk_offset[i] + offset_wrt_chunk[i]); + naccessed *= sub_size[i]; /* Offset into mem buffer */ - sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + sub_offset_m[i] = chunk_offset[i] + offset_wrt_chunk[i] + offset_m[i] - offset_f[i]; } - + /* - * If we are writing a partial chunk then load the chunk from disk - * and uncompress it if it exists. + * Lock the chunk, copy from application to chunk, then unlock the + * chunk. */ - if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || - !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) { - if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 && - H5F_addr_defined (&(udata.addr))) { - - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, - chunk)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress data"); - } - } else { - assert (chunk_size==udata.key.nbytes); - if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, - chunk)<0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset(chunk, 0, chunk_size); - } + if (NULL==(chunk=H5F_istore_lock (f, layout, comp, chunk_offset, + naccessed==chunk_size, &idx_hint))) { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to read raw data chunk"); } - - /* Transfer data to the chunk */ H5V_hyper_copy(layout->ndims, sub_size, layout->dim, offset_wrt_chunk, chunk, size_m, sub_offset_m, buf); - - /* Compress the chunk */ - if (compressed && - (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) && - nbytes<chunk_size) { - outbuf = compressed; - } else { - outbuf = chunk; - nbytes = chunk_size; + if (H5F_istore_unlock (f, layout, comp, TRUE, chunk_offset, &idx_hint, + chunk, naccessed)<0) { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "uanble to unlock raw data chunk"); } - /* - * Create the chunk it if it doesn't exist, or reallocate the chunk - * if its size changed. Then write the data into the file. - */ - H5F_addr_undef(&(udata.addr)); - udata.key.nbytes = nbytes; - if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, - "unable to allocate chunk"); - } - if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) { - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "unable to write raw storage chunk"); - } - /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i]; @@ -985,12 +1458,8 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, } if (carry) break; } - ret_value = SUCCEED; - done: - H5MM_xfree(chunk); - H5MM_xfree (compressed); - FUNC_LEAVE(ret_value); + FUNC_LEAVE(SUCCEED); } @@ -1046,6 +1515,61 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) /*------------------------------------------------------------------------- + * Function: H5F_istore_stats + * + * Purpose: Print raw data cache statistics to the stderr stream. If + * HEADERS is non-zero then print table column headers, + * otherwise assume that the H5AC layer has already printed them. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_stats (H5F_t *f, hbool_t headers) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + double miss_rate; + char ascii[32]; + + FUNC_ENTER (H5F_istore_stats, FAIL); + + if (headers) { + fprintf(stderr, "H5F: raw data cache statistics for file %s\n", + f->name); + fprintf(stderr, " %-18s %8s %8s %8s %8s+%-8s\n", + "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes"); + fprintf(stderr, " %-18s %8s %8s %8s %8s-%-8s\n", + "-----", "----", "------", "--------", "-----", "-------"); + } + + if (rdcc->nhits>0 || rdcc->nmisses>0) { + miss_rate = 100.0 * rdcc->nmisses / + (rdcc->nhits + rdcc->nmisses); + } else { + miss_rate = 0.0; + } + if (miss_rate > 100) { + sprintf(ascii, "%7d%%", (int) (miss_rate + 0.5)); + } else { + sprintf(ascii, "%7.2f%%", miss_rate); + } + + fprintf(stderr, " %-18s %8u %8u %7s %8d+%-9ld\n", + "raw data chunks", rdcc->nhits, rdcc->nmisses, ascii, + rdcc->ninits, (long)(rdcc->nflushes)-(long)(rdcc->ninits)); + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_debug * * Purpose: Debugs a B-tree node for indexed raw data storage. @@ -97,7 +97,8 @@ static intn interface_initialize_g = FALSE; static void H5F_term_interface(void); /* PRIVATE PROTOTYPES */ -static H5F_t *H5F_new(H5F_file_t *shared); +static H5F_t *H5F_new(H5F_file_t *shared, const H5F_create_t *fcpl, + const H5F_access_t *fapl); static herr_t H5F_dest(H5F_t *f); static herr_t H5F_flush(H5F_t *f, hbool_t invalidate); static herr_t H5F_locate_signature(H5F_low_t *f_handle, @@ -141,6 +142,9 @@ H5F_init_interface(void) } /* Initialize the default file access property list */ + H5F_access_dflt.mdc_nelmts = H5AC_NSLOTS; + H5F_access_dflt.rdcc_nbytes = 1024*1024; /*1MB*/ + H5F_access_dflt.rdcc_w0 = 0.75; /*preempt fully read chunks*/ H5F_access_dflt.driver = H5F_LOW_DFLT; #if (H5F_LOW_DFLT == H5F_LOW_SEC2) /* Nothing to initialize */ @@ -468,7 +472,8 @@ H5Fis_hdf5(const char *filename) * H5Fopen and H5Fcreate functions then fill in various * fields. If SHARED is a non-null pointer then the shared info * to which it points has the reference count incremented. - * Otherwise a new, empty shared info struct is created. + * Otherwise a new, empty shared info struct is created and + * initialized with the specified file access property list. * * Errors: * @@ -485,9 +490,11 @@ H5Fis_hdf5(const char *filename) *------------------------------------------------------------------------- */ static H5F_t * -H5F_new(H5F_file_t *shared) +H5F_new(H5F_file_t *shared, const H5F_create_t *fcpl, const H5F_access_t *fapl) { - H5F_t *f = NULL; + H5F_t *f = NULL; + intn n; + FUNC_ENTER(H5F_new, NULL); f = H5MM_xcalloc(1, sizeof(H5F_t)); @@ -500,13 +507,39 @@ H5F_new(H5F_file_t *shared) H5F_addr_undef(&(f->shared->freespace_addr)); H5F_addr_undef(&(f->shared->hdf5_eof)); - /* Create a main cache */ - H5AC_create(f, H5AC_NSLOTS); + /* + * Deep-copy the file creation and file access property lists into + * the new file handle. We do this early because some values might + * need to change as the file is being opened. + */ + if (NULL==(f->shared->create_parms=H5P_copy(H5P_FILE_CREATE, fcpl))) { + HRETURN_ERROR (H5E_FILE, H5E_CANTINIT, NULL, + "unable to copy file creation property list"); + } + if (NULL==(f->shared->access_parms=H5P_copy(H5P_FILE_ACCESS, fapl))) { + HRETURN_ERROR (H5E_FILE, H5E_CANTINIT, NULL, + "unable to copy file access property list"); + } + + /* + * Create a meta data cache with the specified number of elements. + * The cache might be created with a different number of elements and + * the access property list should be updated to reflect that. + */ + if ((n=H5AC_create(f, f->shared->access_parms->mdc_nelmts))<0) { + HRETURN_ERROR (H5E_FILE, H5E_CANTINIT, NULL, + "unable to create meta data cache"); + } + f->shared->access_parms->mdc_nelmts = n; + + /* Create the chunk cache */ + H5F_istore_init (f); } f->shared->nrefs++; FUNC_LEAVE(f); } + /*------------------------------------------------------------------------- * Function: H5F_dest @@ -547,6 +580,10 @@ H5F_dest(H5F_t *f) HERROR (H5E_FILE, H5E_CANTINIT, "problems closing file"); ret_value = FAIL; /*but keep going*/ } + if (H5F_istore_dest (f)<0) { + HERROR (H5E_FILE, H5E_CANTINIT, "problems closing file"); + ret_value = FAIL; /*but keep going*/ + } f->shared->cwfs = H5MM_xfree (f->shared->cwfs); H5P_close (H5P_FILE_CREATE, f->shared->create_parms); H5P_close (H5P_FILE_ACCESS, f->shared->access_parms); @@ -735,7 +772,7 @@ H5F_open(const char *name, uintn flags, old->shared->flags |= H5F_ACC_RDWR; fd = NULL; /*so we don't close it during error */ } - f = H5F_new(old->shared); + f = H5F_new(old->shared, NULL, NULL); } else if (flags & H5F_ACC_TRUNC) { /* Truncate existing file */ @@ -749,7 +786,7 @@ H5F_open(const char *name, uintn flags, HRETURN_ERROR(H5E_FILE, H5E_CANTCREATE, NULL, "can't truncate file"); } - f = H5F_new(NULL); + f = H5F_new(NULL, create_parms, access_parms); f->shared->key = search; f->shared->flags = flags; f->shared->lf = fd; @@ -762,7 +799,7 @@ H5F_open(const char *name, uintn flags, HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "cannot open existing file"); } - f = H5F_new(NULL); + f = H5F_new(NULL, create_parms, access_parms); f->shared->key = search; f->shared->flags = flags; f->shared->lf = fd; @@ -794,7 +831,7 @@ H5F_open(const char *name, uintn flags, HRETURN_ERROR(H5E_FILE, H5E_CANTCREATE, NULL, "can't create file"); } - f = H5F_new(NULL); + f = H5F_new(NULL, create_parms, access_parms); f->shared->key = search; f->shared->flags = flags; f->shared->lf = fd; @@ -816,13 +853,10 @@ H5F_open(const char *name, uintn flags, f->name = H5MM_xstrdup(name); /* - * Update the file creation parameters and file access parameters with - * default values if this is the first time this file is opened. Some of - * the properties may need to be updated. + * Some of the properties may need to be updated. We would like to + * eventually get rid of this step by not having redundant data! */ if (1 == f->shared->nrefs) { - f->shared->create_parms = H5P_copy (H5P_FILE_CREATE, create_parms); - f->shared->access_parms = H5P_copy (H5P_FILE_ACCESS, access_parms); if (H5F_LOW_FAMILY==f->shared->access_parms->driver) { haddr_t x = f->shared->lf->u.fam.memb_size; f->shared->access_parms->u.fam.memb_size = x; @@ -1255,12 +1289,20 @@ H5F_flush(H5F_t *f, hbool_t invalidate) * once for read-only and once for read-write, and then calling * H5F_flush() with the read-only handle, still causes data to be flushed. */ - if (0 == (H5F_ACC_RDWR & f->shared->flags)) + if (0 == (H5F_ACC_RDWR & f->shared->flags)) { HRETURN(SUCCEED); + } - /* flush (and invalidate) the entire cache */ + /* flush the entire raw data cache */ + if (H5F_istore_flush (f)<0) { + HRETURN_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, + "unable to flush raw data cache"); + } + + /* flush (and invalidate) the entire meta data cache */ if (H5AC_flush(f, NULL, 0, invalidate) < 0) { - HRETURN_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache"); + HRETURN_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, + "unable to flush meta data cache"); } /* encode the file boot block */ @@ -1371,7 +1413,10 @@ H5F_close(H5F_t *f) } /* Dump debugging info */ - if (f->intent & H5F_ACC_DEBUG) H5AC_debug(f); + if (f->intent & H5F_ACC_DEBUG) { + H5AC_debug(f); + H5F_istore_stats (f, FALSE); + } /* Close files and release resources */ H5F_low_close(f->shared->lf, f->shared->access_parms); diff --git a/src/H5Fistore.c b/src/H5Fistore.c index 7e80f1a..91c4e9d 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -2,8 +2,17 @@ * Copyright (C) 1997 NCSA * All rights reserved. * - * Programmer: Robb Matzke <matzke@llnl.gov> - * Wednesday, October 8, 1997 + * Programmer: Robb Matzke <matzke@llnl.gov> + * Wednesday, October 8, 1997 + * + * Purpose: Indexed (chunked) I/O functions. The logical + * multi-dimensional data space is regularly partitioned into + * same-sized "chunks", the first of which is aligned with the + * logical origin. The chunks are given a multi-dimensional + * index which is used as a lookup key in a B-tree that maps + * chunk index to disk address. Each chunk can be compressed + * independently and the chunks may move around in the file as + * their storage requirements change. */ #include <H5private.h> #include <H5Dprivate.h> @@ -14,13 +23,25 @@ #include <H5Oprivate.h> #include <H5Vprivate.h> - /* Interface initialization */ #define PABLO_MASK H5F_istore_mask static hbool_t interface_initialize_g = FALSE; #define INTERFACE_INIT NULL -/* PRIVATE PROTOTYPES */ +/* Raw data chunks are cached. Each entry in the cache is: */ +typedef struct H5F_rdcc_ent_t { + hbool_t locked; /*entry is locked in cache */ + hbool_t dirty; /*needs to be written to disk? */ + H5O_layout_t *layout; /*the layout message */ + H5O_compress_t *comp; /*compression message */ + hssize_t offset[H5O_LAYOUT_NDIMS]; /*chunk name */ + size_t chunk_size; /*size of a chunk */ + size_t rd_count; /*bytes remaining to be read */ + size_t wr_count; /*bytes remaining to be written */ + uint8 *chunk; /*the uncompressed chunk data */ +} H5F_rdcc_ent_t; + +/* Private prototypes */ static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata); static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key, void *_udata, void *_rt_key, haddr_t *); @@ -546,7 +567,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, * Already exists. If the new size is not the same as the old size * then we should reallocate storage. */ -#if 1 if (lt_key->nbytes != udata->key.nbytes) { if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr, udata->key.nbytes, new_node/*out*/)<0) { @@ -561,13 +581,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, udata->addr = *addr; ret_value = H5B_INS_NOOP; } -#else - assert (lt_key->nbytes == udata->key.nbytes); - assert (!H5F_addr_defined (&(udata->addr)) || - H5F_addr_eq (&(udata->addr), addr)); - udata->addr = *addr; - ret_value = H5B_INS_NOOP; -#endif } else if (H5V_hyper_disjointp(udata->mesg.ndims, lt_key->offset, udata->mesg.dim, @@ -606,6 +619,596 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- + * Function: H5F_istore_init + * + * Purpose: Initialize the raw data chunk cache for a file. This is + * called when the file handle is initialized. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Monday, May 18, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_init (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + + FUNC_ENTER (H5F_istore_init, FAIL); + + HDmemset (rdcc, 0, sizeof(H5F_rdcc_t)); + if (f->shared->access_parms->rdcc_nbytes>0) { + rdcc->nslots = 25; /*some initial number of slots*/ + rdcc->slot = H5MM_xcalloc (rdcc->nslots, sizeof(H5F_rdcc_ent_t)); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_flush_entry + * + * Purpose: Writes a chunk to disk. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_flush_entry (H5F_t *f, H5F_rdcc_ent_t *ent) +{ + void *c_buf = NULL; /*temp compression buffer */ + void *out_ptr = NULL; /*ptr to output buffer */ + size_t nbytes; /*size of output buffer */ + herr_t ret_value = FAIL; /*return value */ + H5F_istore_ud1_t udata; /*pass through B-tree */ + intn i; + + FUNC_ENTER (H5F_istore_flush_entry, FAIL); + assert (ent); + assert (!ent->locked); + if (!ent->dirty) HRETURN (SUCCEED); + + /* Should the chunk be compressed before writing it to disk? */ + if (ent->comp && H5Z_NONE!=ent->comp->method) { + c_buf = H5MM_xmalloc (ent->chunk_size); + nbytes = H5Z_compress (ent->comp, ent->chunk_size, ent->chunk, c_buf); + if (nbytes && nbytes<ent->chunk_size) { + out_ptr = c_buf; + } else { + out_ptr = ent->chunk; + nbytes = ent->chunk_size; + } + } else { + out_ptr = ent->chunk; + nbytes = ent->chunk_size; + } + + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk if its + * size changed. Then write the data into the file. + */ + udata.mesg = *(ent->layout); + H5F_addr_undef(&(udata.addr)); + udata.key.nbytes = nbytes; + for (i=0; i<ent->layout->ndims; i++) { + udata.key.offset[i] = ent->offset[i]; + } + + if (H5B_insert(f, H5B_ISTORE, &(ent->layout->addr), &udata)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to allocate chunk"); + } + if (H5F_block_write (f, &(udata.addr), nbytes, out_ptr)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to write raw data to file"); + } + + /* Mark cache entry as clean */ + ent->dirty = FALSE; + f->shared->rdcc.nflushes++; + ret_value = SUCCEED; + + done: + H5MM_xfree (c_buf); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_flush + * + * Purpose: Writes all dirty chunks to disk but does not remove them from + * the cache. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_flush (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + intn i, nerrors=0; + + FUNC_ENTER (H5F_istore_flush, FAIL); + + for (i=0; i<rdcc->nused; i++) { + if (H5F_istore_flush_entry (f, rdcc->slot+i)<0) { + nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to flush one or more raw data chunks"); + } + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_preempt + * + * Purpose: Preempts the specified entry from the cache, flushing it to + * disk if necessary. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_preempt (H5F_t *f, intn idx) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = rdcc->slot + idx; + + FUNC_ENTER (H5F_istore_preempt, FAIL); + assert (idx>=0 && idx<rdcc->nused); + assert (!ent->locked); + + if (ent->dirty) H5F_istore_flush_entry (f, ent); + H5O_free (H5O_LAYOUT, ent->layout); + H5O_free (H5O_COMPRESS, ent->comp); + H5MM_xfree (ent->chunk); + rdcc->nused -= 1; + rdcc->nbytes -= ent->chunk_size; + HDmemmove (rdcc->slot+idx, rdcc->slot+idx+1, + (rdcc->nused-idx) * sizeof(H5F_rdcc_ent_t)); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_dest + * + * Purpose: Destroy the entire chunk cache by flushing dirty entries, + * preempting all entries, and freeing the cache itself. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_dest (H5F_t *f) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + intn i, nerrors=0; + + FUNC_ENTER (H5F_istore_dest, FAIL); + + for (i=rdcc->nused-1; i>=0; --i) { + if (H5F_istore_flush_entry (f, rdcc->slot+i)<0) { + nerrors++; + } + if (H5F_istore_preempt (f, i)<0) { + nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to flush one or more raw data chunks"); + } + + H5MM_xfree (rdcc->slot); + HDmemset (rdcc, 0, sizeof(H5F_rdcc_t)); + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune + * + * Purpose: Prune the cache by preempting some things until the cache has + * room for something which is SIZE bytes. Only unlocked + * entries are considered for preemption. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_prune (H5F_t *f, size_t size) +{ + intn i, meth0, meth1, nerrors=0; + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent0, *ent1; + double w0 = f->shared->access_parms->rdcc_w0; + size_t total = f->shared->access_parms->rdcc_nbytes; + + FUNC_ENTER (H5F_istore_prune, FAIL); + + /* + * We have two pointers that slide down the cache beginning at the least + * recently used entry. The distance between the pointers represents the + * relative weight. A weight of 50% for the first pointer means that the + * second pointer is half the cache length behind the first pointer. + */ + meth0 = rdcc->nused; + meth1 = rdcc->nused * (1.0+w0); + for (i=MAX(meth0, meth1)-1; + rdcc->nbytes+size>total && i>=0; + --i, --meth0, --meth1) { + + ent0 = rdcc->slot+meth0; /*might be a bad pointer!*/ + ent1 = rdcc->slot+meth1; /*might be a bad pointer!*/ + + if (meth0>=0 && meth0<rdcc->nused && !ent0->locked && + (0==ent0->rd_count || ent0->chunk_size==ent0->rd_count) && + (0==ent0->wr_count || ent0->chunk_size==ent0->wr_count)) { + /* + * Method 0: Preempt entries that have a zero rd_count. If the + * application is accessing a dataset with a set of + * non-overlapping partial I/O requests then chunks with a zero + * rd_count will probably not be accessed in the near future. + */ + if (H5F_istore_preempt (f, meth0)<0) nerrors++; + + } else if (meth1>=0 && meth1<rdcc->nused && !ent1->locked) { + /* + * Method 1: Discard the least recently used members from the + * cache. This is a catch-all. + */ + if (H5F_istore_preempt (f, meth1)<0) nerrors++; + } + } + if (nerrors) { + HRETURN_ERROR (H5E_IO, H5E_CANTFLUSH, FAIL, + "unable to preempt one or more raw data cache entry"); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_lock + * + * Purpose: Return a pointer to an uncompressed chunk. The pointer + * points directly into the chunk cache and should not be freed + * by the caller but will be valid until it is unlocked. The + * input value IDX_HINT is used to speed up cache lookups and + * it's output value should be given to H5F_rdcc_unlock(). + * + * If RELAX is non-zero and the chunk isn't in the cache then + * don't try to read it from the file, but just allocate an + * uninitialized buffer to hold the result. This is indented + * for output functions that are about to overwrite the entire + * chunk. + * + * Return: Success: Ptr to an uncompressed chunk. + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5F_istore_lock (H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, const hssize_t offset[], + hbool_t relax, intn *idx_hint/*in,out*/) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = NULL; + intn i, j, found = -1; + H5F_istore_ud1_t udata; /*B-tree pass-through */ + size_t chunk_size; /*size of a chunk */ + herr_t status; /*func return status */ + void *chunk=NULL; /*the uncompressed chunk*/ + void *temp=NULL; /*temporary chunk buffer*/ + void *ret_value=NULL; /*return value */ + + FUNC_ENTER (H5F_istore_lock, NULL); + + /* First use the hint */ + if (idx_hint && *idx_hint>=0 && *idx_hint<rdcc->nused) { + ent = rdcc->slot + *idx_hint; + if (layout->ndims==ent->layout->ndims) { + for (i=0, found=*idx_hint; found>=0 && i<ent->layout->ndims; i++) { + if (offset[i]!=ent->offset[i]) found = -1; + } + } + } + + /* Then look at all the entries */ + for (i=0; found<0 && i<rdcc->nused; i++) { + ent = rdcc->slot + i; + if (layout->ndims==ent->layout->ndims) { + for (j=0, found=i; found>=0 && j<ent->layout->ndims; j++) { + if (offset[j]!=ent->offset[j]) found = -1; + } + } + } + + + if (found>=0) { + /* + * Already in the cache. Count a hit. + */ + rdcc->nhits++; + + } else if (found<0 && relax) { + /* + * Not in the cache, but we're about to overwrite the whole thing + * anyway, so just allocate a buffer for it but don't initialize that + * buffer with the file contents. Count this as a hit instead of a + * miss because we saved ourselves lots of work. + */ + rdcc->nhits++; + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc (chunk_size); + + } else { + /* + * Not in the cache. Read it from the file and count this as a miss + * if it's in the file or an init if it isn't. + */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + udata.key.offset[i] = offset[i]; + chunk_size *= layout->dim[i]; + } + udata.mesg = *layout; + H5F_addr_undef (&(udata.addr)); + status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); + chunk = H5MM_xmalloc (chunk_size); + if (status>=0 && H5F_addr_defined (&(udata.addr))) { + /* + * The chunk exists on disk but might be compressed. Instead of + * allocating the exact size for the compressed chunk we allocate + * the entire chunk size -- it reduces strain on the malloc() + * subsystem. + */ + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, chunk)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, + "unable to read raw data chunk"); + } + if (udata.key.nbytes<chunk_size) { + temp = H5MM_xmalloc (chunk_size); + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + chunk, chunk_size, temp)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, + "unable to uncompress raw data chunk"); + } + H5MM_xfree (chunk); + chunk = temp; + temp = NULL; + } + rdcc->nmisses++; + } else { + /* + * The chunk doesn't exist in the file. Assume all zeros. + */ + HDmemset (chunk, 0, chunk_size); + rdcc->ninits++; + } + } + + if (found<0 && chunk_size<=f->shared->access_parms->rdcc_nbytes) { + /* + * Add the chunk to the beginning of the cache after pruning the cache + * to make room. + */ + if (H5F_istore_prune (f, chunk_size)<0) { + H5E_clear (); + } + if (rdcc->nused>=rdcc->nslots) { + rdcc->nslots = MAX (25, 2*rdcc->nslots); + rdcc->slot = H5MM_xrealloc (rdcc->slot, + (rdcc->nslots* + sizeof(H5F_rdcc_ent_t))); + } + HDmemmove (rdcc->slot+1, rdcc->slot, + rdcc->nused*sizeof(H5F_rdcc_ent_t)); + rdcc->nused++; + rdcc->nbytes += chunk_size; + ent = rdcc->slot; + ent->locked = 0; + ent->dirty = FALSE; + ent->chunk_size = chunk_size; + ent->layout = H5O_copy (H5O_LAYOUT, layout); + ent->comp = H5O_copy (H5O_COMPRESS, comp); + for (i=0; i<layout->ndims; i++) { + ent->offset[i] = offset[i]; + } + ent->rd_count = chunk_size; + ent->wr_count = chunk_size; + ent->chunk = chunk; + found = 0; + + } else if (found<0) { + /* + * The chunk is larger than the entire cache so we don't cache it. + * This is the reason all those arguments have to be repeated for the + * unlock function. + */ + ent = NULL; + found = -999; + + } else if (found>0) { + /* + * The chunk is not at the beginning of the cache; move it forward by + * one slot. This is how we implement the LRU preemption algorithm. + */ + H5F_rdcc_ent_t x = rdcc->slot[found]; + rdcc->slot[found] = rdcc->slot[found-1]; + rdcc->slot[found-1] = x; + ent = rdcc->slot + --found; + } + + /* Lock the chunk into the cache */ + if (ent) { + assert (!ent->locked); + ent->locked = TRUE; + if (idx_hint) *idx_hint = found; + chunk = ent->chunk; + } + + ret_value = chunk; + done: + if (!ret_value) H5MM_xfree (chunk); + H5MM_xfree (temp); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_unlock + * + * Purpose: Unlocks a previously locked chunk. The LAYOUT, COMP, and + * OFFSET arguments should be the same as for H5F_rdcc_lock(). + * The DIRTY argument should be set to non-zero if the chunk has + * been modified since it was locked. The IDX_HINT argument is + * the returned index hint from the lock operation and BUF is + * the return value from the lock. + * + * The NACCESSED argument should be the number of bytes accessed + * for reading or writing (depending on the value of DIRTY). + * It's only purpose is to provide additional information to the + * preemption policy. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_unlock (H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, hbool_t dirty, + const hssize_t offset[], intn *idx_hint, + uint8 *chunk, size_t naccessed) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + H5F_rdcc_ent_t *ent = NULL; + intn i, found = -1; + + FUNC_ENTER (H5F_istore_unlock, FAIL); + + /* First look at the hint */ + if (idx_hint && *idx_hint>=0 && *idx_hint<rdcc->nused) { + if (rdcc->slot[*idx_hint].chunk==chunk) found = *idx_hint; + } + + /* Then look at all the entries */ + for (i=0; found<0 && i<rdcc->nused; i++) { + if (rdcc->slot[i].chunk==chunk) found = i; + } + + if (found<0) { + /* + * It's not in the cache, probably because it's too big. If it's + * dirty then flush it to disk. In any case, free the chunk. + * Note: we have to copy the layout and compression messages so we + * don't discard the `const' qualifier. + */ + if (dirty) { + H5F_rdcc_ent_t x; + HDmemset (&x, 0, sizeof x); + x.dirty = TRUE; + x.layout = H5O_copy (H5O_LAYOUT, layout); + x.comp = H5O_copy (H5O_COMPRESS, comp); + for (i=0; i<layout->ndims; i++) { + x.offset[i] = offset[i]; + } + x.chunk = chunk; + H5F_istore_flush_entry (f, &x); + H5O_free (H5O_LAYOUT, x.layout); + H5O_free (H5O_COMPRESS, x.comp); + } + H5MM_xfree (chunk); + } else { + /* + * It's in the cache so unlock it. + */ + ent = rdcc->slot + found; + assert (ent->locked); + if (dirty) { + ent->dirty = TRUE; + ent->wr_count -= MIN (ent->wr_count, naccessed); + } else { + ent->rd_count -= MIN (ent->rd_count, naccessed); + } + ent->locked = FALSE; + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_read * * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw @@ -629,18 +1232,17 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, { hssize_t offset_m[H5O_LAYOUT_NDIMS]; hsize_t size_m[H5O_LAYOUT_NDIMS]; - intn i, carry; hsize_t idx_cur[H5O_LAYOUT_NDIMS]; hsize_t idx_min[H5O_LAYOUT_NDIMS]; hsize_t idx_max[H5O_LAYOUT_NDIMS]; hsize_t sub_size[H5O_LAYOUT_NDIMS]; hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; - size_t chunk_size; - uint8 *chunk=NULL, *compressed=NULL; - H5F_istore_ud1_t udata; - herr_t status; - herr_t ret_value = FAIL; + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; + intn i, carry; + size_t naccessed; /*bytes accessed in chnk*/ + uint8 *chunk=NULL; /*ptr to a chunk buffer */ + intn idx_hint=0; /*cache index hint */ FUNC_ENTER(H5F_istore_read, FAIL); @@ -672,61 +1274,10 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, } #endif - /* Determine the chunk size and allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - if (comp && H5Z_NONE!=comp->method) { - compressed = H5MM_xmalloc (chunk_size); - } - - /* - * As a special case if the source is aligned on a chunk boundary and is - * the same size as a chunk, and the destination is the same size as a - * chunk, then instead of reading into a temporary buffer and then into - * the destination, we read directly into the destination. - */ - for (i=0; i<layout->ndims; i++) { - if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ - if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ - if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ - udata.key.offset[i] = offset_f[i]; - } - if (i==layout->ndims) { - udata.mesg = *layout; - H5F_addr_undef (&(udata.addr)); - status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined (&(udata.addr))) { - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, buf)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress raw storage chunk"); - } - } else { - assert (udata.key.nbytes==chunk_size); - if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset (buf, 0, chunk_size); - } - HGOTO_DONE (SUCCEED); - } - /* - * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing - * or copying each chunk into a temporary buffer, and then copying it to - * it's destination. + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each to its final destination in the + * application buffer. */ for (i=0; i<layout->ndims; i++) { idx_min[i] = offset_f[i] / layout->dim[i]; @@ -734,59 +1285,42 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, idx_cur[i] = idx_min[i]; } - /* Initialize non-changing part of udata */ - udata.mesg = *layout; - /* Loop over all chunks */ while (1) { - - for (i=0; i<layout->ndims; i++) { + for (i=0, naccessed=1; i<layout->ndims; i++) { /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); - udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + chunk_offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - - udata.key.offset[i]; + offset_wrt_chunk[i] = MAX(offset_f[i], chunk_offset[i]) - + chunk_offset[i]; sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], offset_f[i]+size[i]) - - (udata.key.offset[i] + offset_wrt_chunk[i]); + (chunk_offset[i] + offset_wrt_chunk[i]); + naccessed *= sub_size[i]; /* Offset into mem buffer */ - sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + sub_offset_m[i] = chunk_offset[i] + offset_wrt_chunk[i] + offset_m[i] - offset_f[i]; } - /* Read chunk */ - H5F_addr_undef(&(udata.addr)); - status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined(&(udata.addr))) { - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, - chunk)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress data"); - } - } else { - assert (udata.key.nbytes == chunk_size); - if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset(chunk, 0, chunk_size); + /* + * Lock the chunk, transfer data to the application, then unlock the + * chunk. + */ + if (NULL==(chunk=H5F_istore_lock (f, layout, comp, chunk_offset, + FALSE, &idx_hint))) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw data chunk"); } - - /* Transfer data from the chunk buffer to the application */ H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m, - (void *)buf, layout->dim, offset_wrt_chunk, chunk); + (void*)buf, layout->dim, offset_wrt_chunk, chunk); + if (H5F_istore_unlock (f, layout, comp, FALSE, chunk_offset, &idx_hint, + chunk, naccessed)<0) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to unlock raw data chunk"); + } /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { @@ -795,12 +1329,7 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, } if (carry) break; } - ret_value = SUCCEED; - - done: - H5MM_xfree(chunk); - H5MM_xfree (compressed); - FUNC_LEAVE(ret_value); + FUNC_LEAVE(SUCCEED); } @@ -833,12 +1362,12 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, hsize_t idx_min[H5O_LAYOUT_NDIMS]; hsize_t idx_max[H5O_LAYOUT_NDIMS]; hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; - hsize_t chunk_size, nbytes; - uint8 *chunk=NULL, *compressed=NULL, *outbuf; - H5F_istore_ud1_t udata; - herr_t ret_value = FAIL; + uint8 *chunk=NULL; + intn idx_hint=0; + size_t chunk_size, naccessed; FUNC_ENTER(H5F_istore_write, FAIL); @@ -855,9 +1384,10 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, * For now the source must not be a hyperslab. It must be an entire * memory buffer. */ - for (i=0; i<layout->ndims; i++) { + for (i=0, chunk_size=1; i<layout->ndims; i++) { offset_m[i] = 0; size_m[i] = size[i]; + chunk_size *= layout->dim[i]; } #ifndef NDEBUG @@ -871,10 +1401,9 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, #endif /* - * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks copying - * each chunk into a temporary buffer, compressing or decompressing, and - * then copying it to it's destination. + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each chunk from the application to the + * chunk cache. */ for (i=0; i<layout->ndims; i++) { idx_min[i] = offset_f[i] / layout->dim[i]; @@ -882,102 +1411,46 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, idx_cur[i] = idx_min[i]; } - /* Allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - if (comp && H5Z_NONE!=comp->method) { - compressed = H5MM_xmalloc (chunk_size); - } - - /* Initialize non-changing part of udata */ - udata.mesg = *layout; /* Loop over all chunks */ while (1) { - - for (i=0; i<layout->ndims; i++) { + + for (i=0, naccessed=1; i<layout->ndims; i++) { /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); - udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + chunk_offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - - udata.key.offset[i]; + offset_wrt_chunk[i] = MAX(offset_f[i], chunk_offset[i]) - + chunk_offset[i]; sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], offset_f[i]+size[i]) - - (udata.key.offset[i] + offset_wrt_chunk[i]); + (chunk_offset[i] + offset_wrt_chunk[i]); + naccessed *= sub_size[i]; /* Offset into mem buffer */ - sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + sub_offset_m[i] = chunk_offset[i] + offset_wrt_chunk[i] + offset_m[i] - offset_f[i]; } - + /* - * If we are writing a partial chunk then load the chunk from disk - * and uncompress it if it exists. + * Lock the chunk, copy from application to chunk, then unlock the + * chunk. */ - if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || - !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) { - if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 && - H5F_addr_defined (&(udata.addr))) { - - if (compressed && udata.key.nbytes<chunk_size) { - if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, - compressed)<0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, - compressed, chunk_size, - chunk)) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, - "unable to uncompress data"); - } - } else { - assert (chunk_size==udata.key.nbytes); - if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, - chunk)<0) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, - "unable to read raw storage chunk"); - } - } - } else { - HDmemset(chunk, 0, chunk_size); - } + if (NULL==(chunk=H5F_istore_lock (f, layout, comp, chunk_offset, + naccessed==chunk_size, &idx_hint))) { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to read raw data chunk"); } - - /* Transfer data to the chunk */ H5V_hyper_copy(layout->ndims, sub_size, layout->dim, offset_wrt_chunk, chunk, size_m, sub_offset_m, buf); - - /* Compress the chunk */ - if (compressed && - (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) && - nbytes<chunk_size) { - outbuf = compressed; - } else { - outbuf = chunk; - nbytes = chunk_size; + if (H5F_istore_unlock (f, layout, comp, TRUE, chunk_offset, &idx_hint, + chunk, naccessed)<0) { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "uanble to unlock raw data chunk"); } - /* - * Create the chunk it if it doesn't exist, or reallocate the chunk - * if its size changed. Then write the data into the file. - */ - H5F_addr_undef(&(udata.addr)); - udata.key.nbytes = nbytes; - if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, - "unable to allocate chunk"); - } - if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) { - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "unable to write raw storage chunk"); - } - /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i]; @@ -985,12 +1458,8 @@ H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, } if (carry) break; } - ret_value = SUCCEED; - done: - H5MM_xfree(chunk); - H5MM_xfree (compressed); - FUNC_LEAVE(ret_value); + FUNC_LEAVE(SUCCEED); } @@ -1046,6 +1515,61 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) /*------------------------------------------------------------------------- + * Function: H5F_istore_stats + * + * Purpose: Print raw data cache statistics to the stderr stream. If + * HEADERS is non-zero then print table column headers, + * otherwise assume that the H5AC layer has already printed them. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_stats (H5F_t *f, hbool_t headers) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + double miss_rate; + char ascii[32]; + + FUNC_ENTER (H5F_istore_stats, FAIL); + + if (headers) { + fprintf(stderr, "H5F: raw data cache statistics for file %s\n", + f->name); + fprintf(stderr, " %-18s %8s %8s %8s %8s+%-8s\n", + "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes"); + fprintf(stderr, " %-18s %8s %8s %8s %8s-%-8s\n", + "-----", "----", "------", "--------", "-----", "-------"); + } + + if (rdcc->nhits>0 || rdcc->nmisses>0) { + miss_rate = 100.0 * rdcc->nmisses / + (rdcc->nhits + rdcc->nmisses); + } else { + miss_rate = 0.0; + } + if (miss_rate > 100) { + sprintf(ascii, "%7d%%", (int) (miss_rate + 0.5)); + } else { + sprintf(ascii, "%7.2f%%", miss_rate); + } + + fprintf(stderr, " %-18s %8u %8u %7s %8d+%-9ld\n", + "raw data chunks", rdcc->nhits, rdcc->nmisses, ascii, + rdcc->ninits, (long)(rdcc->nflushes)-(long)(rdcc->ninits)); + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_debug * * Purpose: Debugs a B-tree node for indexed raw data storage. diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index cce26e1..26942ce 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -238,6 +238,9 @@ typedef struct H5F_create_t { * File-access property list. */ typedef struct H5F_access_t { + intn mdc_nelmts; /* Size of meta data cache (nelmts) */ + size_t rdcc_nbytes; /* Size of raw data chunk cache (bytes) */ + double rdcc_w0; /* Preempt read chunks first? [0.0..1.0]*/ H5F_driver_t driver; /* Low level file driver */ union { @@ -386,6 +389,18 @@ extern const H5F_low_class_t H5F_LOW_SPLIT_g[]; /* Split meta/raw data */ extern const H5F_low_class_t H5F_LOW_MPIO_g[]; /* MPI-IO */ #endif +/* The raw data chunk cache */ +typedef struct H5F_rdcc_t { + uintn ninits; /* Number of chunk creations */ + uintn nhits; /* Number of cache hits */ + uintn nmisses;/* Number of cache misses */ + uintn nflushes;/* Number of cache flushes */ + size_t nbytes; /* Current cached raw data in bytes */ + intn nslots; /* Number of chunk slots allocated */ + intn nused; /* Number of chunk slots in use */ + struct H5F_rdcc_ent_t *slot; /* Chunk slots, each points to a chunk */ +} H5F_rdcc_t; + /* * Define the structure to store the file information for HDF5 files. One of * these structures is allocated per file, not per H5Fopen(). @@ -406,6 +421,7 @@ typedef struct H5F_file_t { struct H5G_t *root_grp; /* Open root group */ intn ncwfs; /* Num entries on cwfs list */ struct H5HG_heap_t **cwfs; /* Global heap cache */ + H5F_rdcc_t rdcc; /* Raw data chunk cache */ } H5F_file_t; /* @@ -494,6 +510,10 @@ herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, const hssize_t file_offset[], const void *_buf); /* Functions that operate on indexed storage */ +herr_t H5F_istore_init (H5F_t *f); +herr_t H5F_istore_flush (H5F_t *f); +herr_t H5F_istore_dest (H5F_t *f); +herr_t H5F_istore_stats (H5F_t *f, hbool_t headers); herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); herr_t H5F_istore_read(H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_compress_t *comp, @@ -1,59 +1,61 @@ /*------------------------------------------------------------------------- - * Copyright (C) 1997 National Center for Supercomputing Applications. - * All rights reserved. + * Copyright (C) 1997 National Center for Supercomputing Applications. + * All rights reserved. * *------------------------------------------------------------------------- * - * Created: H5MM.c - * Jul 10 1997 - * Robb Matzke <matzke@llnl.gov> + * Created: H5MM.c + * Jul 10 1997 + * Robb Matzke <matzke@llnl.gov> * - * Purpose: Memory management functions. + * Purpose: Memory management functions. * - * Modifications: + * Modifications: * *------------------------------------------------------------------------- */ #include <H5private.h> #include <H5MMprivate.h> + /*------------------------------------------------------------------------- - * Function: H5MM_xmalloc + * Function: H5MM_xmalloc * - * Purpose: Just like malloc(3) except it aborts on an error. + * Purpose: Just like malloc(3) except it aborts on an error. * - * Return: Success: Ptr to new memory. + * Return: Success: Ptr to new memory. * - * Failure: abort() + * Failure: abort() * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Jul 10 1997 + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 * * Modifications: * *------------------------------------------------------------------------- */ -void * +void * H5MM_xmalloc(size_t size) { - void *mem = HDmalloc(size); + void *mem = HDmalloc(size); assert(mem); return mem; } + /*------------------------------------------------------------------------- - * Function: H5MM_xcalloc + * Function: H5MM_xcalloc * - * Purpose: Just like calloc(3) except it aborts on an error. + * Purpose: Just like calloc(3) except it aborts on an error. * - * Return: Success: Ptr to memory. + * Return: Success: Ptr to memory. * - * Failure: abort() + * Failure: abort() * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Jul 10 1997 + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 * * Modifications: * @@ -73,64 +75,65 @@ H5MM_xcalloc(intn n, size_t size) return mem; } + /*------------------------------------------------------------------------- - * Function: H5MM_xrealloc + * Function: H5MM_xrealloc * - * Purpose: Just like the POSIX version of realloc(3) exept it aborts - * on an error. Specifically, the following calls are - * equivalent + * Purpose: Just like the POSIX version of realloc(3) exept it aborts + * on an error. Specifically, the following calls are + * equivalent * - * H5MM_xrealloc (NULL, size) <==> H5MM_xmalloc (size) - * H5MM_xrealloc (ptr, 0) <==> H5MM_xfree (ptr) - * H5MM_xrealloc (NULL, 0) <==> NULL + * H5MM_xrealloc (NULL, size) <==> H5MM_xmalloc (size) + * H5MM_xrealloc (ptr, 0) <==> H5MM_xfree (ptr) + * H5MM_xrealloc (NULL, 0) <==> NULL * - * Return: Success: Ptr to new memory or NULL if the memory - * was freed. + * Return: Success: Ptr to new memory or NULL if the memory + * was freed. * - * Failure: abort() + * Failure: abort() * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Jul 10 1997 + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 * * Modifications: * *------------------------------------------------------------------------- */ -void * +void * H5MM_xrealloc(void *mem, size_t size) { if (!mem) { - if (0 == size) - return NULL; - mem = H5MM_xmalloc(size); + if (0 == size) return NULL; + mem = H5MM_xmalloc(size); } else if (0 == size) { - mem = H5MM_xfree(mem); + mem = H5MM_xfree(mem); } else { - mem = HDrealloc(mem, size); - assert(mem); + mem = HDrealloc(mem, size); + assert(mem); } return mem; } + /*------------------------------------------------------------------------- - * Function: H5MM_strdup + * Function: H5MM_strdup * - * Purpose: Duplicates a string. If the string to be duplicated is the - * null pointer, then return null. If the string to be duplicated - * is the empty string then return a new empty string. + * Purpose: Duplicates a string. If the string to be duplicated is the + * null pointer, then return null. If the string to be duplicated + * is the empty string then return a new empty string. * - * Return: Success: Ptr to a new string (or null if no string). + * Return: Success: Ptr to a new string (or null if no string). * - * Failure: abort() + * Failure: abort() * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Jul 10 1997 + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 * * Modifications: * @@ -139,30 +142,31 @@ H5MM_xrealloc(void *mem, size_t size) char * H5MM_xstrdup(const char *s) { - char *mem; + char *mem; if (!s) return NULL; mem = H5MM_xmalloc(HDstrlen(s) + 1); HDstrcpy(mem, s); return mem; } + /*------------------------------------------------------------------------- - * Function: H5MM_xfree + * Function: H5MM_xfree * - * Purpose: Just like free(3) except null pointers are allowed as - * arguments, and the return value (always NULL) can be - * assigned to the pointer whose memory was just freed: + * Purpose: Just like free(3) except null pointers are allowed as + * arguments, and the return value (always NULL) can be + * assigned to the pointer whose memory was just freed: * - * thing = H5MM_xfree (thing); + * thing = H5MM_xfree (thing); * - * Return: Success: NULL + * Return: Success: NULL * - * Failure: never fails + * Failure: never fails * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Jul 10 1997 + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 * * Modifications: * @@ -667,6 +667,76 @@ H5O_reset(const H5O_class_t *type, void *native) /*------------------------------------------------------------------------- + * Function: H5O_free + * + * Purpose: Similar to H5O_reset() except it also frees the message + * pointer. + * + * Return: Success: NULL + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void * +H5O_free (const H5O_class_t *type, void *mesg) +{ + FUNC_ENTER (H5O_free, NULL); + + if (mesg) { + H5O_reset (type, mesg); + H5MM_xfree (mesg); + } + + FUNC_LEAVE (NULL); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_copy + * + * Purpose: Copies a message. If MESG is is the null pointer then a null + * pointer is returned with no error. + * + * Return: Success: Ptr to the new message + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Thursday, May 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void * +H5O_copy (const H5O_class_t *type, const void *mesg) +{ + void *ret_value = NULL; + + FUNC_ENTER (H5O_copy, NULL); + + assert (type); + assert (type->copy); + + if (mesg) { + if (NULL==(ret_value=(type->copy)(mesg, NULL))) { + HRETURN_ERROR (H5E_OHDR, H5E_CANTINIT, NULL, + "unable to copy object header message"); + } + } + + FUNC_LEAVE (ret_value); +} + + + +/*------------------------------------------------------------------------- * Function: H5O_link * * Purpose: Adjust the link count for an object header by adding @@ -1209,9 +1279,7 @@ H5O_remove(H5G_entry_t *ent, const H5O_class_t *type, intn sequence) /* change message type to nil and zero it */ oh->mesg[i].type = H5O_NULL; HDmemset(oh->mesg[i].raw, 0, oh->mesg[i].raw_size); - H5O_reset(type, oh->mesg[i].native); - oh->mesg[i].native = H5MM_xfree(oh->mesg[i].native); - + oh->mesg[i].native = H5O_free (type, oh->mesg[i].native); oh->mesg[i].dirty = TRUE; oh->dirty = TRUE; } @@ -1850,8 +1918,7 @@ H5O_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, (oh->mesg[i].type->debug)(f, mesg, stream, indent+3, MAX (0, fwidth-3)); } - H5O_reset (oh->mesg[i].type, mesg); - H5MM_xfree (mesg); + H5O_free (oh->mesg[i].type, mesg); H5MM_xfree (p); } } diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index b6e340b..7db9313 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -247,6 +247,8 @@ intn H5O_modify (H5G_entry_t *ent, const H5O_class_t *type, intn overwrite, uintn flags, const void *mesg); herr_t H5O_remove (H5G_entry_t *ent, const H5O_class_t *type, intn sequence); herr_t H5O_reset (const H5O_class_t *type, void *native); +void *H5O_free (const H5O_class_t *type, void *mesg); +void *H5O_copy (const H5O_class_t *type, const void *mesg); herr_t H5O_share (H5F_t *f, const H5O_class_t *type, const void *mesg, H5HG_t *hobj/*out*/); herr_t H5O_debug (H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, @@ -59,7 +59,7 @@ H5P_init_interface(void) FUNC_ENTER(H5P_init_interface, FAIL); /* - * Make sure the file creation and file access default templates are + * Make sure the file creation and file access default property lists are * initialized since this might be done at run-time instead of compile * time. */ @@ -71,9 +71,9 @@ H5P_init_interface(void) assert(H5P_NCLASSES <= H5_TEMPLATE_MAX - H5_TEMPLATE_0); /* - * Initialize the mappings between template classes and atom groups. We - * keep the two separate because template classes are publicly visible but - * atom groups aren't. + * Initialize the mappings between property list classes and atom + * groups. We keep the two separate because property list classes are + * publicly visible but atom groups aren't. */ for (i = 0; i < H5P_NCLASSES; i++) { status = H5I_init_group((H5I_group_t)(H5_TEMPLATE_0 +i), @@ -127,20 +127,23 @@ H5P_term_interface(void) NAME H5Pcreate PURPOSE - Returns a copy of the default template for some class of templates. + Returns a copy of the default property list for some class of property + * lists. USAGE herr_t H5Pcreate (type) - H5P_class_t type; IN: Template class whose default is desired. + H5P_class_t type; IN: Property list class whose default is + * desired. RETURNS - Template ID or FAIL + Property list ID or FAIL ERRORS - ARGS BADVALUE Unknown template class. - ATOM CANTINIT Can't register template. + ARGS BADVALUE Unknown property list class. + ATOM CANTINIT Can't register property list. INTERNAL UNSUPPORTED Not implemented yet. DESCRIPTION - Returns a copy of the default template for some class of templates. + Returns a copy of the default property list for some class of property + * lists. --------------------------------------------------------------------------*/ hid_t H5Pcreate(H5P_class_t type) @@ -150,7 +153,7 @@ H5Pcreate(H5P_class_t type) FUNC_ENTER(H5Pcreate, FAIL); - /* Allocate a new template and initialize it with default values */ + /* Allocate a new property list and initialize it with default values */ switch (type) { case H5P_FILE_CREATE: tmpl = H5MM_xmalloc(sizeof(H5F_create_t)); @@ -174,13 +177,13 @@ H5Pcreate(H5P_class_t type) default: HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "unknown template class"); + "unknown property list class"); } - /* Atomize the new template */ + /* Atomize the new property list */ if ((ret_value = H5P_create(type, tmpl)) < 0) { HRETURN_ERROR(H5E_ATOM, H5E_CANTINIT, FAIL, - "can't register template"); + "unable to register property list"); } FUNC_LEAVE(ret_value); } @@ -188,11 +191,12 @@ H5Pcreate(H5P_class_t type) /*------------------------------------------------------------------------- * Function: H5P_create * - * Purpose: Given a pointer to some template struct, atomize the template - * and return its ID. The template memory is not copied, so the - * caller should not free it; it will be freed by H5P_release(). + * Purpose: Given a pointer to some property list struct, atomize the + * property list and return its ID. The property list memory is + * not copied, so the caller should not free it; it will be + * freed by H5P_release(). * - * Return: Success: A new template ID. + * Return: Success: A new property list ID. * * Failure: FAIL * @@ -214,10 +218,10 @@ H5P_create(H5P_class_t type, void *tmpl) assert(type >= 0 && type < H5P_NCLASSES); assert(tmpl); - /* Atomize the new template */ + /* Atomize the new property list */ if ((ret_value=H5I_register((H5I_group_t)(H5_TEMPLATE_0+type), tmpl)) < 0) { HRETURN_ERROR(H5E_ATOM, H5E_CANTINIT, FAIL, - "can't register template"); + "unable to register property list"); } FUNC_LEAVE(ret_value); @@ -227,14 +231,14 @@ H5P_create(H5P_class_t type, void *tmpl) NAME H5Pclose PURPOSE - Release access to a template object. + Release access to a property list object. USAGE herr_t H5Pclose(oid) - hid_t oid; IN: Template object to release access to + hid_t oid; IN: property list object to release access to RETURNS SUCCEED/FAIL DESCRIPTION - This function releases access to a template object + This function releases access to a property list object --------------------------------------------------------------------------*/ herr_t H5Pclose(hid_t tid) @@ -265,8 +269,8 @@ H5Pclose(hid_t tid) /*------------------------------------------------------------------------- * Function: H5P_close * - * Purpose: Closes a template and frees the memory associated with the - * template. + * Purpose: Closes a property list and frees the memory associated with + * the property list. * * Return: Success: SUCCEED * @@ -290,7 +294,7 @@ H5P_close (H5P_class_t type, void *tmpl) /* Check args */ if (!tmpl) HRETURN (SUCCEED); - /* Some templates may need to do special things */ + /* Some property lists may need to do special things */ switch (type) { case H5P_FILE_ACCESS: switch (fa_list->driver) { @@ -340,7 +344,7 @@ H5P_close (H5P_class_t type, void *tmpl) "unknown property list class"); } - /* Free the template struct and return */ + /* Free the property list struct and return */ H5MM_xfree(tmpl); FUNC_LEAVE(SUCCEED); } @@ -349,9 +353,9 @@ H5P_close (H5P_class_t type, void *tmpl) /*------------------------------------------------------------------------- * Function: H5Pget_class * - * Purpose: Returns the class identifier for a template. + * Purpose: Returns the class identifier for a property list. * - * Return: Success: A template class + * Return: Success: A property list class * * Failure: H5P_NO_CLASS (-1) * @@ -375,7 +379,8 @@ H5Pget_class(hid_t tid) group >= H5_TEMPLATE_MAX || #endif group < H5_TEMPLATE_0) { - HRETURN_ERROR(H5E_ATOM, H5E_BADATOM, H5P_NO_CLASS, "not a template"); + HRETURN_ERROR(H5E_ATOM, H5E_BADATOM, H5P_NO_CLASS, + "not a property list"); } ret_value = (H5P_class_t)(group - H5_TEMPLATE_0); FUNC_LEAVE(ret_value); @@ -419,7 +424,7 @@ H5Pget_version(hid_t tid, int *boot/*out*/, int *freelist/*out*/, if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Get values */ if (boot) *boot = tmpl->bootblock_ver; @@ -433,7 +438,8 @@ H5Pget_version(hid_t tid, int *boot/*out*/, int *freelist/*out*/, /*------------------------------------------------------------------------- * Function: H5Pset_userblock * - * Purpose: Sets the userblock size field of a file creation template. + * Purpose: Sets the userblock size field of a file creation property + * list. * * Return: Success: SUCCEED * @@ -458,7 +464,7 @@ H5Pset_userblock(hid_t tid, hsize_t size) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } for (i=8; i<8*sizeof(hsize_t); i++) { hsize_t p2 = 8==i ? 0 : ((hsize_t)1<<i); @@ -477,7 +483,8 @@ H5Pset_userblock(hid_t tid, hsize_t size) /*------------------------------------------------------------------------- * Function: H5Pget_userblock * - * Purpose: Queries the size of a user block in a file creation template. + * Purpose: Queries the size of a user block in a file creation property + * list. * * Return: Success: SUCCEED, size returned through SIZE argument. * @@ -501,7 +508,7 @@ H5Pget_userblock(hid_t tid, hsize_t *size) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Get value */ if (size) *size = tmpl->userblock_size; @@ -512,9 +519,9 @@ H5Pget_userblock(hid_t tid, hsize_t *size) /*------------------------------------------------------------------------- * Function: H5Pset_sizes * - * Purpose: Sets file size-of addresses and sizes. TEMPLATE - * should be a file creation template. A value of zero causes - * the property to not change. + * Purpose: Sets file size-of addresses and sizes. TID should be a file + * creation property list. A value of zero causes the property + * to not change. * * Return: Success: SUCCEED * @@ -538,7 +545,7 @@ H5Pset_sizes(hid_t tid, size_t sizeof_addr, size_t sizeof_size) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } if (sizeof_addr) { if (sizeof_addr != 2 && sizeof_addr != 4 && @@ -567,8 +574,8 @@ H5Pset_sizes(hid_t tid, size_t sizeof_addr, size_t sizeof_size) * Function: H5Pget_sizes * * Purpose: Returns the size of address and size quantities stored in a - * file according to a file creation template. Either (or even - * both) SIZEOF_ADDR and SIZEOF_SIZE may be null pointers. + * file according to a file creation property list. Either (or + * even both) SIZEOF_ADDR and SIZEOF_SIZE may be null pointers. * * Return: Success: SUCCEED, sizes returned through arguments. * @@ -593,7 +600,7 @@ H5Pget_sizes(hid_t tid, if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Get values */ if (sizeof_addr) @@ -644,7 +651,7 @@ H5Pset_sym_k(hid_t tid, int ik, int lk) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Set values */ if (ik > 0) { @@ -686,7 +693,7 @@ H5Pget_sym_k(hid_t tid, int *ik /*out */ , int *lk /*out */ ) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Get values */ if (ik) @@ -726,7 +733,7 @@ H5Pset_istore_k(hid_t tid, int ik) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } if (ik <= 0) { HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -767,7 +774,7 @@ H5Pget_istore_k(hid_t tid, int *ik /*out */ ) if (H5P_FILE_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file creation template"); + "not a file creation property list"); } /* Get value */ if (ik) @@ -803,7 +810,7 @@ H5Pset_layout(hid_t tid, H5D_layout_t layout) if (H5P_DATASET_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a dataset creation template"); + "not a dataset creation property list"); } if (layout < 0 || layout >= H5D_NLAYOUTS) { HRETURN_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, @@ -818,7 +825,7 @@ H5Pset_layout(hid_t tid, H5D_layout_t layout) /*------------------------------------------------------------------------- * Function: H5Pget_layout * - * Purpose: Retrieves layout type of a dataset creation template. + * Purpose: Retrieves layout type of a dataset creation property list. * * Return: Success: The layout type * @@ -842,7 +849,7 @@ H5Pget_layout(hid_t tid) if (H5P_DATASET_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, H5D_LAYOUT_ERROR, - "not a dataset creation template"); + "not a dataset creation property list"); } FUNC_LEAVE(tmpl->layout); } @@ -880,7 +887,7 @@ H5Pset_chunk(hid_t tid, int ndims, const hsize_t dim[]) if (H5P_DATASET_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a dataset creation template"); + "not a dataset creation property list"); } if (ndims <= 0) { HRETURN_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, @@ -1200,7 +1207,7 @@ H5Pset_stdio (hid_t tid) if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } /* Set driver */ @@ -1280,7 +1287,7 @@ H5Pset_sec2 (hid_t tid) if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } /* Set driver */ @@ -1364,7 +1371,7 @@ H5Pset_core (hid_t tid, size_t increment) if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } if (increment<1) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1458,19 +1465,19 @@ H5Pset_split (hid_t tid, const char *meta_ext, hid_t meta_tid, if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } if (H5P_DEFAULT!=meta_tid && (H5P_FILE_ACCESS != H5Pget_class(meta_tid) || NULL == (meta_tmpl = H5I_object(meta_tid)))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } if (H5P_DEFAULT!=raw_tid && (H5P_FILE_ACCESS != H5Pget_class(raw_tid) || NULL == (raw_tmpl = H5I_object(raw_tid)))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } /* Set driver */ @@ -1603,7 +1610,7 @@ H5Pset_family (hid_t tid, hsize_t memb_size, hid_t memb_tid) if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } if (memb_size && memb_size<1024) { HRETURN_ERROR (H5E_ARGS, H5E_BADRANGE, FAIL, @@ -1613,7 +1620,7 @@ H5Pset_family (hid_t tid, hsize_t memb_size, hid_t memb_tid) (H5P_FILE_ACCESS != H5Pget_class(memb_tid) || NULL == (tmpl = H5I_object(memb_tid)))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } /* Set driver */ @@ -1682,6 +1689,106 @@ H5Pget_family (hid_t tid, hsize_t *memb_size/*out*/, hid_t *memb_tid/*out*/) /*------------------------------------------------------------------------- + * Function: H5Pset_cache + * + * Purpose: Set the number of objects in the meta data cache and the + * total number of bytes in the raw data chunk cache. + * + * The RDCC_W0 value should be between 0 and 1 inclusive and + * indicates how much chunks that have been fully read are + * favored for preemption. A value of zero means fully read + * chunks are treated no differently than other chunks (the + * preemption is strictly LRU) while a value of one means fully + * read chunks are always preempted before other chunks. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Tuesday, May 19, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_cache (hid_t tid, int mdc_nelmts, size_t rdcc_nbytes, + double rdcc_w0) +{ + H5F_access_t *fapl = NULL; + + FUNC_ENTER (H5Pset_cache, FAIL); + + /* Check arguments */ + if (H5P_FILE_ACCESS!=H5Pget_class (tid) || + NULL==(fapl=H5I_object (tid))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access property list"); + } + if (mdc_nelmts<0) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "meta data cache size must be non-negative"); + } + if (rdcc_w0<0.0 || rdcc_w0>1.0) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "raw data cache w0 value must be between 0.0 and 1.0 " + "inclusive"); + } + + /* Set sizes */ + fapl->mdc_nelmts = mdc_nelmts; + fapl->rdcc_nbytes = rdcc_nbytes; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_cache + * + * Purpose: Retrieves the maximum possible number of elements in the meta + * data cache and the maximum possible number of bytes in the + * raw data chunk cache. Any (or all) arguments may be null + * pointers in which case the corresponding datum is not + * returned. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Tuesday, May 19, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_cache (hid_t tid, int *mdc_nelmts, size_t *rdcc_nbytes, + double *rdcc_w0) +{ + H5F_access_t *fapl = NULL; + + FUNC_ENTER (H5Pget_cache, FAIL); + + /* Check arguments */ + if (H5P_FILE_ACCESS!=H5Pget_class (tid) || + NULL==(fapl=H5I_object (tid))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access property list"); + } + + /* Get sizes */ + if (mdc_nelmts) *mdc_nelmts = fapl->mdc_nelmts; + if (rdcc_nbytes) *rdcc_nbytes = fapl->rdcc_nbytes; + if (rdcc_w0) *rdcc_w0 = fapl->rdcc_w0; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5Pset_buffer * * Purpose: Given a dataset transfer property list, set the maximum size @@ -1889,7 +1996,7 @@ H5Pset_compression (hid_t plist_id, H5Z_method_t method, unsigned int flags, HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list"); } - if (method<0 || method>H5Z_MAXVAL) { + if (method<0 || method>H5Z_USERDEF_MAX) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid compression method"); } @@ -2054,27 +2161,28 @@ H5Pget_deflate (hid_t plist_id) * Signature: herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info) * * Purpose: Store the access mode for MPIO call and the user supplied - * communicator and info in the access template which can then - * be used to open file. This function is available only in the - * parallel HDF5 library and is not a collective function. + * communicator and info in the access property list which can + * then be used to open file. This function is available only + * in the parallel HDF5 library and is not a collective + * function. * * Parameters: * hid_t tid - * ID of template to modify + * ID of property list to modify * MPI_Comm comm * MPI communicator to be used for file open as defined in * MPI_FILE_OPEN of MPI-2. This function does not make a * duplicated communicator. Any modification to comm after * this function call returns may have undetermined effect - * to the access template. Users should call this function - * again to setup the template. + * to the access property list. Users should call this + * function again to setup the property list. * MPI_Info info * MPI info object to be used for file open as defined in * MPI_FILE_OPEN of MPI-2. This function does not make a * duplicated info. Any modification to info after * this function call returns may have undetermined effect - * to the access template. Users should call this function - * again to setup the template. + * to the access property list. Users should call this + * function again to setup the property list. * * Return: Success: SUCCEED * @@ -2086,11 +2194,11 @@ H5Pget_deflate (hid_t plist_id) * Modifications: * * Robb Matzke, 18 Feb 1998 - * Check all arguments before the template is updated so we don't leave - * the template in a bad state if something goes wrong. Also, the - * template data type changed to allow more generality so all the - * mpi-related stuff is in the `u.mpi' member. The `access_mode' will - * contain only mpi-related flags defined in H5Fpublic.h. + * Check all arguments before the property list is updated so we don't + * leave the property list in a bad state if something goes wrong. Also, + * the property list data type changed to allow more generality so all + * the mpi-related stuff is in the `u.mpi' member. The `access_mode' + * will contain only mpi-related flags defined in H5Fpublic.h. * * Albert Cheng, Apr 16, 1998 * Removed the access_mode argument. The access_mode is changed @@ -2112,7 +2220,7 @@ H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info) if (H5P_FILE_ACCESS != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a file access template"); + "not a file access property list"); } #ifdef LATER @@ -2286,24 +2394,24 @@ H5Pget_xfer (hid_t tid, H5D_transfer_t *data_xfer_mode) NAME H5Pcopy PURPOSE - Copy a template + Copy a property list USAGE hid_t H5P_copy(tid) - hid_t tid; IN: Template object to copy + hid_t tid; IN: property list object to copy RETURNS - Returns template ID (atom) on success, FAIL on failure + Returns property list ID (atom) on success, FAIL on failure ERRORS - ARGS BADRANGE Unknown template class. - ATOM BADATOM Can't unatomize template. - ATOM CANTREGISTER Register the atom for the new template. + ARGS BADRANGE Unknown property list class. + ATOM BADATOM Can't unatomize property list. + ATOM CANTREGISTER Register the atom for the new property list. INTERNAL UNSUPPORTED Dataset transfer properties are not implemented yet. INTERNAL UNSUPPORTED File access properties are not implemented yet. DESCRIPTION - This function creates a new copy of a template with all the same parameter - settings. + * This function creates a new copy of a property list with all the same + * parameter settings. --------------------------------------------------------------------------*/ hid_t H5Pcopy(hid_t tid) @@ -2321,19 +2429,19 @@ H5Pcopy(hid_t tid) (type = H5Pget_class(tid)) < 0 || (group = H5I_group(tid)) < 0) { HRETURN_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, - "can't unatomize template"); + "unable to unatomize property list"); } /* Copy it */ if (NULL==(new_tmpl=H5P_copy (type, tmpl))) { HRETURN_ERROR (H5E_INTERNAL, H5E_CANTINIT, FAIL, - "unable to copy template"); + "unable to copy property list"); } - /* Register the atom for the new template */ + /* Register the atom for the new property list */ if ((ret_value = H5I_register(group, new_tmpl)) < 0) { HRETURN_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, - "unable to atomize template pointer"); + "unable to atomize property list pointer"); } FUNC_LEAVE(ret_value); } @@ -2341,10 +2449,10 @@ H5Pcopy(hid_t tid) /*------------------------------------------------------------------------- * Function: H5P_copy * - * Purpose: Creates a new template and initializes it with some other - * template. + * Purpose: Creates a new property list and initializes it with some + * other property list. * - * Return: Success: Ptr to new template + * Return: Success: Ptr to new property list * * Failure: NULL * @@ -2368,7 +2476,7 @@ H5P_copy (H5P_class_t type, const void *src) FUNC_ENTER (H5P_copy, NULL); - /* How big is the template */ + /* How big is the property list */ switch (type) { case H5P_FILE_CREATE: size = sizeof(H5F_create_t); @@ -2388,10 +2496,10 @@ H5P_copy (H5P_class_t type, const void *src) default: HRETURN_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, - "unknown template class"); + "unknown property list class"); } - /* Create the new template */ + /* Create the new property list */ dst = H5MM_xmalloc(size); HDmemcpy(dst, src, size); @@ -2446,7 +2554,7 @@ H5P_copy (H5P_class_t type, const void *src) default: HRETURN_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, - "unknown template class"); + "unknown property list class"); } FUNC_LEAVE (dst); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index e9717d1..9a3cda2 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -96,6 +96,10 @@ H5Z_method_t H5Pget_compression (hid_t plist_id, unsigned int *flags/*out*/, void *client_data/*out*/); herr_t H5Pset_deflate (hid_t plist_id, int level); int H5Pget_deflate (hid_t plist_id); +herr_t H5Pset_cache (hid_t plist_id, int mdc_nelmts, size_t rdcc_nbytes, + double rdcc_w0); +herr_t H5Pget_cache (hid_t plist_id, int *mdc_nelmts, size_t *rdcc_nbytes, + double *rdcc_w0); #ifdef HAVE_PARALLEL herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info); @@ -50,7 +50,7 @@ typedef struct H5Z_class_t { } uncomp; #endif } H5Z_class_t; -static H5Z_class_t H5Z_g[H5Z_MAXVAL+1]; +static H5Z_class_t H5Z_g[H5Z_USERDEF_MAX+1]; /* Compression and uncompression methods */ static size_t H5Z_zlib_c (unsigned int flags, size_t __unused__ cd_size, @@ -114,21 +114,21 @@ H5Z_term_interface (void) int i, nprint=0; char name[16]; - for (i=0; i<=H5Z_MAXVAL; i++) { + for (i=0; i<=H5Z_USERDEF_MAX; i++) { if (H5Z_g[i].comp.nbytes || H5Z_g[i].uncomp.nbytes) { if (0==nprint++) { HDfprintf (stderr, "H5Z: compression statistics accumulated " "over life of library:\n"); - HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n", + HDfprintf (stderr, " %-10s %10s %7s %7s %8s %8s %8s %9s\n", "Method", "Total", "Overrun", "Errors", "User", "System", "Elapsed", "Bandwidth"); - HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n", + HDfprintf (stderr, " %-10s %10s %7s %7s %8s %8s %8s %9s\n", "------", "-----", "-------", "------", "----", "------", "-------", "---------"); } sprintf (name, "%s-c", H5Z_g[i].name); HDfprintf (stderr, - " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ", + " %-12s %10Hd %7Hd %7Hd %8.2f %8.2f %8.2f ", name, H5Z_g[i].comp.nbytes, H5Z_g[i].comp.over, @@ -145,7 +145,7 @@ H5Z_term_interface (void) sprintf (name, "%s-u", H5Z_g[i].name); HDfprintf (stderr, - " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ", + " %-12s %10Hd %7Hd %7Hd %8.2f %8.2f %8.2f ", name, H5Z_g[i].uncomp.nbytes, H5Z_g[i].uncomp.over, @@ -198,7 +198,7 @@ H5Zregister (H5Z_method_t method, const char *name, H5Z_func_t cfunc, FUNC_ENTER (H5Zregister, FAIL); /* Check args */ - if (method<0 || method>H5Z_MAXVAL) { + if (method<0 || method>H5Z_USERDEF_MAX) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid data compression method number"); } @@ -381,7 +381,7 @@ H5Z_register (H5Z_method_t method, const char *name, H5Z_func_t cfunc, { FUNC_ENTER (H5Z_register, FAIL); - assert (method>=0 && method<=H5Z_MAXVAL); + assert (method>=0 && method<=H5Z_USERDEF_MAX); H5MM_xfree (H5Z_g[method].name); H5Z_g[method].name = H5MM_xstrdup (name); H5Z_g[method].compress = cfunc; diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index daebf89..2ce1326 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -13,24 +13,25 @@ * 15 are defined by the library. Methods 16-255 are user-defined. */ typedef int H5Z_method_t; -#define H5Z_NONE 0 /*no compression, must be zero */ -#define H5Z_DEFLATE 1 /*deflation like gzip */ -#define H5Z_RES_2 2 /*reserved for internal use */ -#define H5Z_RES_3 3 /*reserved for internal use */ -#define H5Z_RES_4 4 /*reserved for internal use */ -#define H5Z_RES_5 5 /*reserved for internal use */ -#define H5Z_RES_6 6 /*reserved for internal use */ -#define H5Z_RES_7 7 /*reserved for internal use */ -#define H5Z_RES_8 8 /*reserved for internal use */ -#define H5Z_RES_9 9 /*reserved for internal use */ -#define H5Z_RES_10 10 /*reserved for internal use */ -#define H5Z_RES_11 11 /*reserved for internal use */ -#define H5Z_RES_12 12 /*reserved for internal use */ -#define H5Z_RES_13 13 /*reserved for internal use */ -#define H5Z_RES_14 14 /*reserved for internal use */ -#define H5Z_RES_15 15 /*reserved for internal use */ +#define H5Z_NONE 0 /*no compression, must be zero */ +#define H5Z_DEFLATE 1 /*deflation like gzip */ +#define H5Z_RES_2 2 /*reserved for internal use */ +#define H5Z_RES_3 3 /*reserved for internal use */ +#define H5Z_RES_4 4 /*reserved for internal use */ +#define H5Z_RES_5 5 /*reserved for internal use */ +#define H5Z_RES_6 6 /*reserved for internal use */ +#define H5Z_RES_7 7 /*reserved for internal use */ +#define H5Z_RES_8 8 /*reserved for internal use */ +#define H5Z_RES_9 9 /*reserved for internal use */ +#define H5Z_RES_10 10 /*reserved for internal use */ +#define H5Z_RES_11 11 /*reserved for internal use */ +#define H5Z_RES_12 12 /*reserved for internal use */ +#define H5Z_RES_13 13 /*reserved for internal use */ +#define H5Z_RES_14 14 /*reserved for internal use */ +#define H5Z_RES_15 15 /*reserved for internal use */ /* user-defined 16-255 */ -#define H5Z_MAXVAL 255 /*maximum compression method ID */ +#define H5Z_USERDEF_MIN 16 /*first user-defined method */ +#define H5Z_USERDEF_MAX 255 /*last user-defined method */ /* * A compression function takes some configuration data which comes from the diff --git a/src/Makefile.in b/src/Makefile.in index 0f7db53..1cffe5a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -18,10 +18,10 @@ PARALLEL_SRC=H5Fmpio.c LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcore.c \ H5Ffamily.c H5Fistore.c H5Flow.c H5Fsec2.c H5Fsplit.c H5Fstdio.c H5G.c \ - H5Gent.c H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c \ - H5O.c H5Oattr.c H5Ocomp.c H5Ocont.c H5Odtype.c H5Oefl.c H5Olayout.c \ - H5Oname.c H5Onull.c H5Osdspace.c H5Oshared.c H5Ostab.c H5P.c H5S.c \ - H5Ssimp.c H5T.c H5Tconv.c H5Tinit.c H5V.c H5Z.c @PARALLEL_SRC@ + H5Gent.c H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c \ + H5Oattr.c H5Ocomp.c H5Ocont.c H5Odtype.c H5Oefl.c H5Olayout.c H5Oname.c \ + H5Onull.c H5Osdspace.c H5Oshared.c H5Ostab.c H5P.c H5S.c H5Ssimp.c H5T.c \ + H5Tconv.c H5Tinit.c H5V.c H5Z.c @PARALLEL_SRC@ LIB_OBJ=$(LIB_SRC:.c=.o) diff --git a/test/.distdep b/test/.distdep index bb1cb84..f2e23cd 100644 --- a/test/.distdep +++ b/test/.distdep @@ -403,3 +403,25 @@ links.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h +chunk.o: \ + chunk.c \ + ../src/hdf5.h \ + ../src/H5public.h \ + ../src/H5config.h \ + ../src/H5Ipublic.h \ + ../src/H5Apublic.h \ + ../src/H5ACpublic.h \ + ../src/H5Bpublic.h \ + ../src/H5Dpublic.h \ + ../src/H5Epublic.h \ + ../src/H5Fpublic.h \ + ../src/H5Gpublic.h \ + ../src/H5HGpublic.h \ + ../src/H5HLpublic.h \ + ../src/H5MFpublic.h \ + ../src/H5MMpublic.h \ + ../src/H5Opublic.h \ + ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ + ../src/H5Spublic.h \ + ../src/H5Tpublic.h diff --git a/test/Makefile.in b/test/Makefile.in index aab4e6c..37a01ac 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -12,10 +12,10 @@ CPPFLAGS=-I. -I../src @CPPFLAGS@ # These are our main targets. They should be listed in the order to be # executed, generally most specific tests to least specific tests. PROGS=testhdf5 gheap hyperslab istore dtypes dsets cmpd_dset extend external \ - shtype iopipe big links + shtype iopipe big links chunk TESTS=testhdf5 gheap hyperslab istore dtypes dsets cmpd_dset extend external \ shtype links -TIMINGS=iopipe +TIMINGS=iopipe chunk # Temporary files MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \ @@ -24,15 +24,15 @@ MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \ extern_2.raw extern_2b.raw extern_3.raw extern_3b.raw \ extern_4.raw extern_4b.raw iopipe.raw iopipe.h5 gheap0.h5 \ gheap1.h5 gheap2.h5 gheap3.h5 gheap4.h5 shtype0.h5 shtype1.h5 \ - shtype2a.h5 shtype2b.h5 shtype3.h5 links.h5 + shtype2a.h5 shtype2b.h5 shtype3.h5 links.h5 chunk.h5 # Source and object files for programs... The PROG_SRC list contains all the # source files and is used for things like dependencies, archiving, etc. The # other source lists are for the individual tests, the files of which may # overlap with other tests. PROG_SRC=testhdf5.c tattr.c tfile.c theap.c tmeta.c tohdr.c tstab.c th5s.c \ - dtypes.c hyperslab.c istore.c dsets.c cmpd_dset.c extend.c external.c \ - iopipe.c gheap.c shtype.c big.c links.c + dtypes.c hyperslab.c istore.c dsets.c cmpd_dset.c extend.c external.c \ + iopipe.c gheap.c shtype.c big.c links.c chunk.c PROG_OBJ=$(PROG_SRC:.c=.o) TESTHDF5_SRC=testhdf5.c tattr.c tfile.c theap.c tmeta.c tohdr.c tstab.c th5s.c @@ -77,6 +77,9 @@ BIG_OBJ=$(BIG_SRC:.c=.o) LINKS_SRC=links.c LINKS_OBJ=$(LINKS_SRC:.c=.o) +CHUNK_SRC=chunk.c +CHUNK_OBJ=$(CHUNK_SRC:.c=.o) + # Private header files (not to be installed)... PRIVATE_HDR=testhdf5.h @@ -133,4 +136,7 @@ big: $(BIG_OBJ) ../src/libhdf5.a links: $(LINKS_OBJ) ../src/libhdf5.a $(CC) $(CFLAGS) -o $@ $(LINKS_OBJ) ../src/libhdf5.a $(LIBS) +chunk: $(CHUNK_OBJ) ../src/libhdf5.a + $(CC) $(CFLAGS) -o $@ $(CHUNK_OBJ) ../src/libhdf5.a $(LIBS) + @CONCLUDE@ @@ -68,7 +68,7 @@ writer (int wrt_n) */ plist = H5Pcreate (H5P_FILE_ACCESS); H5Pset_family (plist, FAMILY_SIZE, H5P_DEFAULT); - file = H5Fcreate (FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, plist); + file = H5Fcreate (FNAME, H5F_ACC_TRUNC|H5F_ACC_DEBUG, H5P_DEFAULT, plist); H5Pclose (plist); /* Create simple data spaces according to the size specified above. */ @@ -133,7 +133,7 @@ reader (const char *script_name) /* Open HDF5 file */ plist = H5Pcreate (H5P_FILE_ACCESS); H5Pset_family (plist, FAMILY_SIZE, H5P_DEFAULT); - file = H5Fopen (FNAME, H5F_ACC_RDONLY, plist); + file = H5Fopen (FNAME, H5F_ACC_RDONLY|H5F_ACC_DEBUG, plist); H5Pclose (plist); /* Open the dataset */ diff --git a/test/chunk.c b/test/chunk.c new file mode 100644 index 0000000..35b5b85 --- /dev/null +++ b/test/chunk.c @@ -0,0 +1,521 @@ +/* + * Copyright (C) 1998 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke <robb@arborea.spizella.com> + * Thursday, May 14, 1998 + * + * Purpose: Checks the effect of various I/O request sizes and raw data + * cache sizes. Performance depends on the amount of data read + * from disk, and we use a trick to get that number: a + * compress/uncompress pair that counts the amount of data read. + * Since the data itself is not important, the compression just + * removes a byte and uncompression adds some byte of arbitrary + * value. The change in size is necessary or the library will + * not store the data as compressed and thus not call the + * uncompression method. + */ +#include <assert.h> +#include <hdf5.h> +#include <stdio.h> +#include <stdlib.h> + +#include <H5config.h> +#ifndef HAVE_ATTRIBUTE +# undef __attribute__ +# define __attribute__(X) /*void*/ +# define __unused__ /*void*/ +#else +# define __unused__ __attribute__((unused)) +#endif + +#define FILE_NAME "chunk.h5" +#define LINESPOINTS "lines" +#define CH_SIZE 100 /*squared in terms of bytes */ +#define DS_SIZE 20 /*squared in terms of chunks */ +#define ZMETHNO H5Z_USERDEF_MIN +#define READ 0 +#define WRITE 1 +#define MIN(X,Y) ((X)<(Y)?(X):(Y)) +#define MAX(X,Y) ((X)>(Y)?(X):(Y)) +#define SQUARE(X) ((X)*(X)) + +/* The row-major test */ +#define RM_CACHE_STRT 25 +#define RM_CACHE_END 25 +#define RM_CACHE_DELT 5 +#define RM_START 0.20 +#define RM_END 5.00 +#define RM_DELTA 0.0025 + +/* Diagonal test */ +#define DIAG_CACHE_STRT 25 +#define DIAG_CACHE_END 25 +#define DIAG_CACHE_DELT 5 +#define DIAG_START 0.20 +#define DIAG_END 5.00 +#define DIAG_DELTA 0.0025 + +static size_t nio_g; +static hid_t fapl_g = -1; + + +/*------------------------------------------------------------------------- + * Function: count_c + * + * Purpose: A bogus compression method that just removes the last byte. + * + * Return: Success: src_nbytes-1 + * + * Failure: never fails + * + * Programmer: Robb Matzke + * Thursday, May 14, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +count_c (unsigned int __unused__ flags, size_t __unused__ cd_size, + const void __unused__ *client_data, size_t src_nbytes, + const void *src, size_t __unused__ dst_nbytes, void *dst/*out*/) +{ + memcpy (dst, src, src_nbytes-1); + nio_g += src_nbytes; + return src_nbytes-1; +} + + +/*------------------------------------------------------------------------- + * Function: count_u + * + * Purpose: A bogus unompress method that just adds a zero to the end. + * + * Return: Success: src_nbytes+1 + * + * Failure: never fails + * + * Programmer: Robb Matzke + * Thursday, May 14, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +count_u (unsigned int __unused__ flags, size_t __unused__ cd_size, + const void __unused__ *client_data, size_t src_nbytes, + const void *src, size_t __unused__ dst_nbytes, void *dst/*out*/) +{ + memcpy (dst, src, src_nbytes); + ((char*)dst)[src_nbytes] = 0; + nio_g += src_nbytes+1; + return src_nbytes+1; +} + + +/*------------------------------------------------------------------------- + * Function: create_dataset + * + * Purpose: Creates a square dataset with square chunks, registers a + * stupid compress/uncompress pair for counting I/O, and + * initializes the dataset. The chunk size is in bytes, the + * dataset size is in terms of chunks. + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, May 14, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void +create_dataset (void) +{ + hid_t file, space, dcpl, dset; + hsize_t size[2]; + char *buf; + + /* The file */ + file = H5Fcreate (FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_g); + + /* The data space */ + size[0] = size[1] = DS_SIZE * CH_SIZE; + space = H5Screate_simple (2, size, size); + + /* The storage layout and compression */ + dcpl = H5Pcreate (H5P_DATASET_CREATE); + size[0] = size[1] = CH_SIZE; + H5Pset_chunk (dcpl, 2, size); + H5Zregister (ZMETHNO, "counter", count_c, count_u); + H5Pset_compression (dcpl, ZMETHNO, 0, 0, NULL); + + /* The dataset */ + dset = H5Dcreate (file, "dset", H5T_NATIVE_CHAR, space, dcpl); + assert (dset>=0); + + /* The data */ + buf = calloc (1, SQUARE (DS_SIZE*CH_SIZE)); + H5Dwrite (dset, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + free (buf); + + /* Close */ + H5Dclose (dset); + H5Sclose (space); + H5Pclose (dcpl); + H5Fclose (file); +} + + +/*------------------------------------------------------------------------- + * Function: test_rowmaj + * + * Purpose: Reads the entire dataset using the specified size-squared + * I/O requests in row major order. + * + * Return: Efficiency: data requested divided by data actually read. + * + * Programmer: Robb Matzke + * Thursday, May 14, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static double +test_rowmaj (int op, hsize_t cache_size, hsize_t io_size) +{ + hid_t file, dset, mem_space, file_space; + char *buf = calloc (1, SQUARE(io_size)); + hsize_t i, j, hs_size[2]; + hssize_t hs_offset[2]; + int mdc_nelmts; + double w0; + + H5Pget_cache (fapl_g, &mdc_nelmts, NULL, &w0); + H5Pset_cache (fapl_g, mdc_nelmts, cache_size*SQUARE (CH_SIZE), w0); + file = H5Fopen (FILE_NAME, H5F_ACC_RDWR, fapl_g); + dset = H5Dopen (file, "dset"); + file_space = H5Dget_space (dset); + nio_g = 0; + + for (i=0; i<CH_SIZE*DS_SIZE; i+=io_size) { +#if 0 + fprintf (stderr, "%5d\b\b\b\b\b", (int)i); + fflush (stderr); +#endif + for (j=0; j<CH_SIZE*DS_SIZE; j+=io_size) { + hs_offset[0] = i; + hs_size[0] = MIN (io_size, CH_SIZE*DS_SIZE-i); + hs_offset[1] = j; + hs_size[1] = MIN (io_size, CH_SIZE*DS_SIZE-j); + mem_space = H5Screate_simple (2, hs_size, hs_size); + H5Sset_hyperslab (file_space, hs_offset, hs_size, NULL); + + if (READ==op) { + H5Dread (dset, H5T_NATIVE_CHAR, mem_space, file_space, + H5P_DEFAULT, buf); + } else { + H5Dwrite (dset, H5T_NATIVE_CHAR, mem_space, file_space, + H5P_DEFAULT, buf); + } + H5Sclose (mem_space); + } + } + + free (buf); + H5Sclose (file_space); + H5Dclose (dset); + H5Fclose (file); + + return (double)SQUARE(CH_SIZE*DS_SIZE)/nio_g; +} + + +/*------------------------------------------------------------------------- + * Function: test_diag + * + * Purpose: Reads windows diagonally across the dataset. Each window is + * offset from the previous window by OFFSET in the x and y + * directions. The reading ends after the (k,k) value is read + * where k is the maximum index in the dataset. + * + * Return: Efficiency. + * + * Programmer: Robb Matzke + * Friday, May 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static double +test_diag (int op, hsize_t cache_size, hsize_t io_size, hsize_t offset) +{ + hid_t file, dset, mem_space, file_space; + hsize_t i, hs_size[2], nio=0; + hssize_t hs_offset[2]; + char *buf = calloc (1, SQUARE (io_size)); + int mdc_nelmts; + double w0; + + H5Pget_cache (fapl_g, &mdc_nelmts, NULL, &w0); + H5Pset_cache (fapl_g, mdc_nelmts, cache_size*SQUARE (CH_SIZE), w0); + file = H5Fopen (FILE_NAME, H5F_ACC_RDWR, fapl_g); + dset = H5Dopen (file, "dset"); + file_space = H5Dget_space (dset); + nio_g = 0; + + for (i=0, hs_size[0]=io_size; hs_size[0]==io_size; i+=offset) { + hs_offset[0] = hs_offset[1] = i; + hs_size[0] = hs_size[1] = MIN (io_size, CH_SIZE*DS_SIZE-i); + mem_space = H5Screate_simple (2, hs_size, hs_size); + H5Sset_hyperslab (file_space, hs_offset, hs_size, NULL); + if (READ==op) { + H5Dread (dset, H5T_NATIVE_CHAR, mem_space, file_space, + H5P_DEFAULT, buf); + } else { + H5Dwrite (dset, H5T_NATIVE_CHAR, mem_space, file_space, + H5P_DEFAULT, buf); + } + H5Sclose (mem_space); + nio += hs_size[0]*hs_size[1]; + if (i>0) nio -= SQUARE (io_size-offset); + } + + free (buf); + H5Sclose (file_space); + H5Dclose (dset); + H5Fclose (file); + + return (double)nio/nio_g; +} + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: See file prologue. + * + * Return: Success: + * + * Failure: + * + * Programmer: Robb Matzke + * Thursday, May 14, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +int +main (void) +{ + hsize_t io_size; + double effic, io_percent; + FILE *f, *d; + int cache_size; + double w0; + + /* + * Create a global file access property list. + */ + fapl_g = H5Pcreate (H5P_FILE_ACCESS); + H5Pget_cache (fapl_g, NULL, NULL, &w0); + + /* Create the file */ + create_dataset (); + f = fopen ("x-gnuplot", "w"); + + +#if 1 + /* + * Test row-major reading of the dataset with various sizes of request + * windows. + */ + if (RM_CACHE_STRT==RM_CACHE_END) { + fprintf (f, "set yrange [0:1.2]\n"); + fprintf (f, "set ytics 0, 0.1, 1\n"); + fprintf (f, "set xlabel \"%s\"\n", + "Request size as a fraction of chunk size"); + fprintf (f, "set ylabel \"Efficiency\"\n"); + fprintf (f, "set title \"Cache %d chunks, w0=%g, " + "Size=(total=%d, chunk=%d)\"\n", + RM_CACHE_STRT, w0, DS_SIZE*CH_SIZE, CH_SIZE); + } else { + fprintf (f, "set autoscale\n"); + fprintf (f, "set hidden3d\n"); + } + + fprintf (f, "set terminal postscript\nset output \"x-rowmaj-rd.ps\"\n"); + fprintf (f, "%s \"x-rowmaj-rd.dat\" title \"RowMaj-Read\" with %s\n", + RM_CACHE_STRT==RM_CACHE_END?"plot":"splot", + LINESPOINTS); + fprintf (f, "set terminal x11\nreplot\n"); + d = fopen ("x-rowmaj-rd.dat", "w"); + for (cache_size=RM_CACHE_STRT; + cache_size<=RM_CACHE_END; + cache_size+=RM_CACHE_DELT) { + for (io_percent=RM_START; io_percent<=RM_END; io_percent+=RM_DELTA) { + io_size = MAX (1, (int)(CH_SIZE*io_percent)); + printf ("Rowmaj-rd %8d %8.2f", cache_size, io_percent); + fflush (stdout); + effic = test_rowmaj (READ, cache_size, io_size); + printf (" %8.2f\n", effic); + if (RM_CACHE_STRT==RM_CACHE_END) { + fprintf (d, "%g %g\n", io_percent, effic); + } else { + fprintf (d, "%g\n", effic); + } + } + fprintf (d, "\n"); + } + fclose (d); + fprintf (f, "pause -1\n"); +#endif + +#if 1 + /* + * Test row-major writing of the dataset with various sizes of request + * windows. + */ + if (RM_CACHE_STRT==RM_CACHE_END) { + fprintf (f, "set yrange [0:1.2]\n"); + fprintf (f, "set ytics 0, 0.1, 1\n"); + fprintf (f, "set xlabel \"%s\"\n", + "Request size as a fraction of chunk size"); + fprintf (f, "set ylabel \"Efficiency\"\n"); + fprintf (f, "set title \"Cache %d chunks,w0=%g, " + "Size=(total=%d, chunk=%d)\"\n", + RM_CACHE_STRT, w0, DS_SIZE*CH_SIZE, CH_SIZE); + } else { + fprintf (f, "set autoscale\n"); + fprintf (f, "set hidden3d\n"); + } + + fprintf (f, "set terminal postscript\nset output \"x-rowmaj-wr.ps\"\n"); + fprintf (f, "%s \"x-rowmaj-wr.dat\" title \"RowMaj-Write\" with %s\n", + RM_CACHE_STRT==RM_CACHE_END?"plot":"splot", + LINESPOINTS); + fprintf (f, "set terminal x11\nreplot\n"); + d = fopen ("x-rowmaj-wr.dat", "w"); + for (cache_size=RM_CACHE_STRT; + cache_size<=RM_CACHE_END; + cache_size+=RM_CACHE_DELT) { + for (io_percent=RM_START; io_percent<=RM_END; io_percent+=RM_DELTA) { + io_size = MAX (1, (int)(CH_SIZE*io_percent)); + printf ("Rowmaj-wr %8d %8.2f", cache_size, io_percent); + fflush (stdout); + effic = test_rowmaj (WRITE, cache_size, io_size); + printf (" %8.2f\n", effic); + if (RM_CACHE_STRT==RM_CACHE_END) { + fprintf (d, "%g %g\n", io_percent, effic); + } else { + fprintf (d, "%g\n", effic); + } + } + fprintf (d, "\n"); + } + fclose (d); + fprintf (f, "pause -1\n"); +#endif + +#if 1 + /* + * Test diagonal read + */ + if (DIAG_CACHE_STRT==DIAG_CACHE_END) { + fprintf (f, "set yrange [0:1.2]\n"); + fprintf (f, "set ytics 0, 0.1, 1\n"); + fprintf (f, "set xlabel \"%s\"\n", + "Request size as a fraction of chunk size"); + fprintf (f, "set ylabel \"Efficiency\"\n"); + fprintf (f, "set title \"Cache %d chunks,w0=%g, " + "Size=(total=%d, chunk=%d)\"\n", + DIAG_CACHE_STRT, w0, DS_SIZE*CH_SIZE, CH_SIZE); + } else { + fprintf (f, "set autoscale\n"); + fprintf (f, "set hidden3d\n"); + } + fprintf (f, "set terminal postscript\nset output \"x-diag-rd.ps\"\n"); + fprintf (f, "%s \"x-diag-rd.dat\" title \"Diag-Read\" with %s\n", + DIAG_CACHE_STRT==DIAG_CACHE_END?"plot":"splot", LINESPOINTS); + fprintf (f, "set terminal x11\nreplot\n"); + d = fopen ("x-diag-rd.dat", "w"); + for (cache_size=DIAG_CACHE_STRT; + cache_size<=DIAG_CACHE_END; + cache_size+=DIAG_CACHE_DELT) { + for (io_percent=DIAG_START; + io_percent<=DIAG_END; + io_percent+=DIAG_DELTA) { + io_size = MAX (1, (int)(CH_SIZE*io_percent)); + printf ("Diag-rd %8d %8.2f", cache_size, io_percent); + fflush (stdout); + effic = test_diag (READ, cache_size, io_size, MAX (1, io_size/2)); + printf (" %8.2f\n", effic); + if (DIAG_CACHE_STRT==DIAG_CACHE_END) { + fprintf (d, "%g %g\n", io_percent, effic); + } else { + fprintf (d, "%g\n", effic); + } + } + fprintf (d, "\n"); + } + fclose (d); + fprintf (f, "pause -1\n"); +#endif + +#if 1 + /* + * Test diagonal write + */ + if (DIAG_CACHE_STRT==DIAG_CACHE_END) { + fprintf (f, "set yrange [0:1.2]\n"); + fprintf (f, "set ytics 0, 0.1, 1\n"); + fprintf (f, "set xlabel \"%s\"\n", + "Request size as a fraction of chunk size"); + fprintf (f, "set ylabel \"Efficiency\"\n"); + fprintf (f, "set title \"Cache %d chunks, w0=%g, " + "Size=(total=%d, chunk=%d)\"\n", + DIAG_CACHE_STRT, w0, DS_SIZE*CH_SIZE, CH_SIZE); + } else { + fprintf (f, "set autoscale\n"); + fprintf (f, "set hidden3d\n"); + } + fprintf (f, "set terminal postscript\nset output \"x-diag-wr.ps\"\n"); + fprintf (f, "%s \"x-diag-wr.dat\" title \"Diag-Write\" with %s\n", + DIAG_CACHE_STRT==DIAG_CACHE_END?"plot":"splot", LINESPOINTS); + fprintf (f, "set terminal x11\nreplot\n"); + d = fopen ("x-diag-wr.dat", "w"); + for (cache_size=DIAG_CACHE_STRT; + cache_size<=DIAG_CACHE_END; + cache_size+=DIAG_CACHE_DELT) { + for (io_percent=DIAG_START; + io_percent<=DIAG_END; + io_percent+=DIAG_DELTA) { + io_size = MAX (1, (int)(CH_SIZE*io_percent)); + printf ("Diag-wr %8d %8.2f", cache_size, io_percent); + fflush (stdout); + effic = test_diag (WRITE, cache_size, io_size, MAX (1, io_size/2)); + printf (" %8.2f\n", effic); + if (DIAG_CACHE_STRT==DIAG_CACHE_END) { + fprintf (d, "%g %g\n", io_percent, effic); + } else { + fprintf (d, "%g\n", effic); + } + } + fprintf (d, "\n"); + } + fclose (d); + fprintf (f, "pause -1\n"); +#endif + + + H5Pclose (fapl_g); + fclose (f); + return 0; +} + diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index edb3cdc..31492b9 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -131,7 +131,7 @@ main (void) hsize_t h_sample[2]; /*hyperslab sampling */ /* Create the file */ - file = H5Fcreate ("cmpd_dset.h5", H5F_ACC_TRUNC, + file = H5Fcreate ("cmpd_dset.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, H5P_DEFAULT, H5P_DEFAULT); assert (file>=0); diff --git a/test/extend.c b/test/extend.c index 09ce489..7ead138 100644 --- a/test/extend.c +++ b/test/extend.c @@ -54,7 +54,8 @@ main (void) assert (mem_space>=0); /* Create the file */ - file = H5Fcreate ("extend.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate ("extend.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, + H5P_DEFAULT, H5P_DEFAULT); assert (file>=0); /* Create the dataset which is originally NX by NY */ diff --git a/test/external.c b/test/external.c index 0694430..49e6d94 100644 --- a/test/external.c +++ b/test/external.c @@ -131,7 +131,8 @@ test_1 (void) * debugging to be emitted before we start playing games with what the * output looks like. */ - file = H5Fcreate ("extern_1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate ("extern_1.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, + H5P_DEFAULT, H5P_DEFAULT); assert (file>=0); grp = H5Gcreate (file, "emit-diagnostics", 8); H5Gclose (grp); @@ -520,7 +521,8 @@ test_2 (void) * debugging to be emitted before we start playing games with what the * output looks like. */ - file = H5Fcreate ("extern_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate ("extern_2.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, + H5P_DEFAULT, H5P_DEFAULT); assert (file>=0); grp = H5Gcreate (file, "emit-diagnostics", 8); H5Gclose (grp); @@ -658,7 +660,8 @@ test_3 (void) /* * Create another file */ - file = H5Fcreate ("extern_3.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate ("extern_3.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, + H5P_DEFAULT, H5P_DEFAULT); assert (file>=0); /* Create the external file list */ diff --git a/test/gheap.c b/test/gheap.c index 4183a84..c4a4420 100644 --- a/test/gheap.c +++ b/test/gheap.c @@ -91,7 +91,8 @@ test_1 (void) /* Open a clean file */ H5Eclear (); - f = H5F_open ("gheap1.h5", H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC, + f = H5F_open ("gheap1.h5", + H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC|H5F_ACC_DEBUG, NULL, NULL); if (!f) { FAILED ("unable to create file"); @@ -133,8 +134,8 @@ test_1 (void) } } - H5F_close (f); puts (" PASSED"); + H5F_close (f); return retval; } @@ -173,7 +174,8 @@ test_2 (void) /* Open a clean file */ H5Eclear (); - f = H5F_open ("gheap2.h5", H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC, + f = H5F_open ("gheap2.h5", + H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC|H5F_ACC_DEBUG, NULL, NULL); if (!f) { FAILED ("unable to create file"); @@ -210,8 +212,8 @@ test_2 (void) } } - H5F_close (f); puts (" PASSED"); + H5F_close (f); return retval; } @@ -249,7 +251,8 @@ test_3 (void) /* Open a clean file */ H5Eclear (); - f = H5F_open ("gheap3.h5", H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC, + f = H5F_open ("gheap3.h5", + H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC|H5F_ACC_DEBUG, NULL, NULL); if (!f) { FAILED ("unable to create file"); @@ -277,8 +280,8 @@ test_3 (void) } } - H5F_close (f); puts (" PASSED"); + H5F_close (f); return retval; } @@ -317,7 +320,8 @@ test_4 (void) /* Open a clean file */ H5Eclear (); - f = H5F_open ("gheap4.h5", H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC, + f = H5F_open ("gheap4.h5", + H5F_ACC_CREAT|H5F_ACC_RDWR|H5F_ACC_TRUNC|H5F_ACC_DEBUG, NULL, NULL); if (!f) { FAILED ("unable to create file"); @@ -352,8 +356,8 @@ test_4 (void) } } - H5F_close (f); puts (" PASSED"); + H5F_close (f); return retval; } |