hdf5.git - Mirror from: https://github.com/HDFGroup/hdf5.git

diff options

author	Frank Baker <fbaker@hdfgroup.org>	2004-08-24 18:57:46 (GMT)
committer	Frank Baker <fbaker@hdfgroup.org>	2004-08-24 18:57:46 (GMT)
commit	644f9c0c98e58eab621d41bf8c59394e6ed64355 (patch)
tree	6d5e6bacd8d88e4da2de5886b2eb8ea87ff0ac29
parent	d7a4ce413acd06e88948ca37ddfabce7909d666a (diff)
download	hdf5-644f9c0c98e58eab621d41bf8c59394e6ed64355.zip hdf5-644f9c0c98e58eab621d41bf8c59394e6ed64355.tar.gz hdf5-644f9c0c98e58eab621d41bf8c59394e6ed64355.tar.bz2

[svn-r9143]

Purpose: Add H5Fget_name Add H5Fget_filesize Minor correction in HTML coding of H5Fget_freespace Platforms tested: Mozilla

Diffstat

-rw-r--r--

doc/html/RM_H5F.html

150

1 files changed, 143 insertions, 7 deletions

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * * the files COPYING and Copyright.html. COPYING can be found at the root * * of the source code distribution tree; Copyright.html can be found at the * * root level of an installed copy of the electronic HDF5 document set and * * is linked from the top-level documents page. It can also be found at * * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* Programmer: Robb Matzke <matzke@llnl.gov> * Wednesday, October 8, 1997 * * Purpose: Indexed (chunked) I/O functions. The logical * multi-dimensional data space is regularly partitioned into * same-sized "chunks", the first of which is aligned with the * logical origin. The chunks are given a multi-dimensional * index which is used as a lookup key in a B-tree that maps * chunk index to disk address. Each chunk can be compressed * independently and the chunks may move around in the file as * their storage requirements change. * * Cache: Disk I/O is performed in units of chunks and H5MF_alloc() * contains code to optionally align chunks on disk block * boundaries for performance. * * The chunk cache is an extendible hash indexed by a function * of storage B-tree address and chunk N-dimensional offset * within the dataset. Collisions are not resolved -- one of * the two chunks competing for the hash slot must be preempted * from the cache. All entries in the hash also participate in * a doubly-linked list and entries are penalized by moving them * toward the front of the list. When a new chunk is about to * be added to the cache the heap is pruned by preempting * entries near the front of the list to make room for the new * entry which is added to the end of the list. */ /****************/ /* Module Setup */ /****************/ #define H5B_PACKAGE /*suppress error about including H5Bpkg */ #define H5D_PACKAGE /*suppress error about including H5Dpkg */ /***********/ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Bpkg.h" /* B-link trees */ #include "H5Dpkg.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File space management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ #include "H5Vprivate.h" /* Vector and array functions */ /****************/ /* Local Macros */ /****************/ /* * Feature: If this constant is defined then every cache preemption and load * causes a character to be printed on the standard error stream: * * `.': Entry was preempted because it has been completely read or * completely written but not partially read and not partially * written. This is often a good reason for preemption because such * a chunk will be unlikely to be referenced in the near future. * * `:': Entry was preempted because it hasn't been used recently. * * `#': Entry was preempted because another chunk collided with it. This * is usually a relatively bad thing. If there are too many of * these then the number of entries in the cache can be increased. * * c: Entry was preempted because the file is closing. * * w: A chunk read operation was eliminated because the library is * about to write new values to the entire chunk. This is a good * thing, especially on files where the chunk size is the same as * the disk block size, chunks are aligned on disk block boundaries, * and the operating system can also eliminate a read operation. */ /*#define H5D_ISTORE_DEBUG */ /* * Given a B-tree node return the dimensionality of the chunks pointed to by * that node. */ #define H5D_ISTORE_NDIMS(X) (((X)->sizeof_rkey-8)/8) #define H5D_HASH(D,ADDR) H5F_addr_hash(ADDR,(D)->cache.chunk.nslots) /******************/ /* Local Typedefs */ /******************/ /* Raw data chunks are cached. Each entry in the cache is: */ typedef struct H5D_rdcc_ent_t { hbool_t locked; /*entry is locked in cache */ hbool_t dirty; /*needs to be written to disk? */ hsize_t offset[H5O_LAYOUT_NDIMS]; /*chunk name */ size_t rd_count; /*bytes remaining to be read */ size_t wr_count; /*bytes remaining to be written */ size_t chunk_size; /*size of a chunk */ size_t alloc_size; /*amount allocated for the chunk */ uint8_t *chunk; /*the unfiltered chunk data */ unsigned idx; /*index in hash table */ struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list */ struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list */ } H5D_rdcc_ent_t; typedef H5D_rdcc_ent_t *H5D_rdcc_ent_ptr_t; /* For free lists */ /* * B-tree key. A key contains the minimum logical N-dimensional address and * the logical size of the chunk to which this key refers. The * fastest-varying dimension is assumed to reference individual bytes of the * array, so a 100-element 1-d array of 4-byte integers would really be a 2-d * array with the slow varying dimension of size 100 and the fast varying * dimension of size 4 (the storage dimensionality has very little to do with * the real dimensionality). * * Only the first few values of the OFFSET and SIZE fields are actually * stored on disk, depending on the dimensionality. * * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5D_istore_key_t { size_t nbytes; /*size of stored data */ hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ unsigned filter_mask; /*excluded filters */ } H5D_istore_key_t; /* * Common data exchange structure for indexed storage nodes. This structure is * passed through the B-link tree layer to the methods for the objects * to which the B-link tree points. */ typedef struct H5D_istore_bt_ud_common_t { /* downward */ H5D_istore_key_t key; /*key values */ const H5O_layout_t *mesg; /*layout message */ } H5D_istore_bt_ud_common_t; /* * Data exchange structure for indexed storage nodes. This structure is * passed through the B-link tree layer to the methods for the objects * to which the B-link tree points for operations which require no * additional information. * * (Just an alias for the "common" info). */ typedef H5D_istore_bt_ud_common_t H5D_istore_ud0_t; typedef struct H5D_istore_ud1_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ haddr_t addr; /*file address of chunk */ } H5D_istore_ud1_t; /* B-tree callback info for iteration to total allocated space */ typedef struct H5D_istore_it_ud1_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ hsize_t total_storage; /*output from iterator */ } H5D_istore_it_ud1_t; /* B-tree callback info for iteration to dump node's info */ typedef struct H5D_istore_it_ud2_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ FILE *stream; /*debug output stream */ hbool_t header_displayed; /* Node's header is displayed? */ } H5D_istore_it_ud2_t; /* B-tree callback info for iteration to prune chunks */ typedef struct H5D_istore_it_ud3_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ hsize_t *dims; /*dataset dimensions */ } H5D_istore_it_ud3_t; /* B-tree callback info for iteration to copy data */ typedef struct H5D_istore_it_ud4_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ H5F_t *file_dst; /* Destination file for copy */ haddr_t addr_dst; /* Address of dest. B-tree */ void *buf; /* Buffer to hold chunk data for read/write */ size_t buf_size; /* Buffer size */ /* needed for converting variable-length data */ hid_t tid_src; /* Datatype ID for source datatype */ hid_t tid_dst; /* Datatype ID for destination datatype */ hid_t tid_mem; /* Datatype ID for memory datatype */ H5T_path_t *tpath_src_mem; /* Datatype conversion path from source file to memory */ H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ void *reclaim_buf; /* Buffer for reclaiming data */ size_t reclaim_buf_size; /* Reclaim buffer size */ size_t nelmts; /* Number of elements in buffer */ H5S_t *buf_space; /* Dataspace describing buffer */ /* needed for compressed variable-length data */ H5O_pline_t *pline; /* Filter pipeline */ } H5D_istore_it_ud4_t; /* B-tree callback info for iteration to obtain chunk address and the index of the chunk for all chunks in the B-tree. */ typedef struct H5D_istore_it_ud5_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ hsize_t* down_chunks; haddr_t *chunk_addr; } H5D_istore_it_ud5_t; /********************/ /* Local Prototypes */ /********************/ static void *H5D_istore_chunk_alloc(size_t size, const H5O_pline_t *pline); static void *H5D_istore_chunk_xfree(void *chk, const H5O_pline_t *pline); static herr_t H5D_istore_shared_create (const H5F_t *f, H5O_layout_t *layout); static herr_t H5D_istore_shared_free (void *page); /* B-tree iterator callbacks */ static int H5D_istore_iter_chunkmap(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, const void *right_key, void *_udata); static int H5D_istore_iter_allocated(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, const void *right_key, void *_udata); static int H5D_istore_iter_dump(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, const void *right_key, void *_udata); static int H5D_istore_prune_extent(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); static int H5D_istore_iter_copy(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); /* B-tree callbacks */ static H5RC_t *H5D_istore_get_shared(const H5F_t *f, const void *_udata); static herr_t H5D_istore_new_node(H5F_t *f, hid_t dxpl_id, H5B_ins_t, void *_lt_key, void *_udata, void *_rt_key, haddr_t *addr_p /*out*/); static int H5D_istore_cmp2(H5F_t *f, hid_t dxpl_id, void *_lt_key, void *_udata, void *_rt_key); static int H5D_istore_cmp3(H5F_t *f, hid_t dxpl_id, void *_lt_key, void *_udata, void *_rt_key); static herr_t H5D_istore_found(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_lt_key, void *_udata); static H5B_ins_t H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, hbool_t *lt_key_changed, void *_md_key, void *_udata, void *_rt_key, hbool_t *rt_key_changed, haddr_t *new_node/*out*/); static H5B_ins_t H5D_istore_remove( H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, hbool_t *lt_key_changed, void *_udata, void *_rt_key, hbool_t *rt_key_changed); static herr_t H5D_istore_decode_key(const H5F_t *f, const H5B_t *bt, const uint8_t *raw, void *_key); static herr_t H5D_istore_encode_key(const H5F_t *f, const H5B_t *bt, uint8_t *raw, void *_key); static herr_t H5D_istore_debug_key(FILE *stream, H5F_t *f, hid_t dxpl_id, int indent, int fwidth, const void *key, const void *udata); /* inherits B-tree like properties from H5B */ H5B_class_t H5B_ISTORE[1] = {{ H5B_ISTORE_ID, /*id */ sizeof(H5D_istore_key_t), /*sizeof_nkey */ H5D_istore_get_shared, /*get_shared */ H5D_istore_new_node, /*new */ H5D_istore_cmp2, /*cmp2 */ H5D_istore_cmp3, /*cmp3 */ H5D_istore_found, /*found */ H5D_istore_insert, /*insert */ FALSE, /*follow min branch? */ FALSE, /*follow max branch? */ H5D_istore_remove, /*remove */ H5D_istore_decode_key, /*decode */ H5D_istore_encode_key, /*encode */ H5D_istore_debug_key, /*debug */ }}; /*********************/ /* Package Variables */ /*********************/ /*****************************/ /* Library Private Variables */ /*****************************/ /*******************/ /* Local Variables */ /*******************/ /* Declare a free list to manage H5F_rdcc_ent_t objects */ H5FL_DEFINE_STATIC(H5D_rdcc_ent_t); /* Declare a free list to manage the H5F_rdcc_ent_ptr_t sequence information */ H5FL_SEQ_DEFINE_STATIC(H5D_rdcc_ent_ptr_t); /* Declare a free list to manage the chunk sequence information */ H5FL_BLK_DEFINE_STATIC(chunk); /* Declare a free list to manage the native key offset sequence information */ H5FL_SEQ_DEFINE_STATIC(size_t); /* Declare a free list to manage the raw page information */ H5FL_BLK_DEFINE_STATIC(chunk_page); /* Declare extern the free list to manage blocks of type conversion data */ H5FL_BLK_EXTERN(type_conv); /*------------------------------------------------------------------------- * Function: H5D_istore_get_shared * * Purpose: Returns the shared B-tree info for the specified UDATA. * * Return: Success: Pointer to the raw B-tree page for this dataset * * Failure: Can't fail * * Programmer: Quincey Koziol * Monday, July 5, 2004 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static H5RC_t * H5D_istore_get_shared(const H5F_t UNUSED *f, const void *_udata) { const H5D_istore_ud0_t *udata = (const H5D_istore_ud0_t *) _udata; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_get_shared) HDassert(udata); HDassert(udata->mesg); HDassert(udata->mesg->u.chunk.btree_shared); /* Increment reference count on B-tree info */ H5RC_INC(udata->mesg->u.chunk.btree_shared); /* Return the pointer to the ref-count object */ FUNC_LEAVE_NOAPI(udata->mesg->u.chunk.btree_shared) } /* end H5D_istore_get_shared() */ /*------------------------------------------------------------------------- * Function: H5D_istore_decode_key * * Purpose: Decodes a raw key into a native key for the B-tree * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * Friday, October 10, 1997 * *------------------------------------------------------------------------- */ static herr_t H5D_istore_decode_key(const H5F_t UNUSED *f, const H5B_t *bt, const uint8_t *raw, void *_key) { H5D_istore_key_t *key = (H5D_istore_key_t *) _key; H5B_shared_t *shared; /* Pointer to shared B-tree info */ size_t ndims; unsigned u; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_decode_key) /* check args */ assert(f); assert(bt); shared=H5RC_GET_OBJ(bt->rc_shared); HDassert(shared); assert(raw); assert(key); ndims = H5D_ISTORE_NDIMS(shared); assert(ndims<=H5O_LAYOUT_NDIMS); /* decode */ UINT32DECODE(raw, key->nbytes); UINT32DECODE(raw, key->filter_mask); for (u=0; u<ndims; u++) UINT64DECODE(raw, key->offset[u]); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D_istore_decode_key() */ /*------------------------------------------------------------------------- * Function: H5D_istore_encode_key * * Purpose: Encode a key from native format to raw format. * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * Friday, October 10, 1997 * *------------------------------------------------------------------------- */ static herr_t H5D_istore_encode_key(const H5F_t UNUSED *f, const H5B_t *bt, uint8_t *raw, void *_key) { H5D_istore_key_t *key = (H5D_istore_key_t *) _key; H5B_shared_t *shared; /* Pointer to shared B-tree info */ size_t ndims; unsigned u; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_encode_key) /* check args */ assert(f); assert(bt); shared=H5RC_GET_OBJ(bt->rc_shared); HDassert(shared); assert(raw); assert(key); ndims = H5D_ISTORE_NDIMS(shared); assert(ndims<=H5O_LAYOUT_NDIMS); /* encode */ UINT32ENCODE(raw, key->nbytes); UINT32ENCODE(raw, key->filter_mask); for (u=0; u<ndims; u++) UINT64ENCODE(raw, key->offset[u]); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D_istore_encode_key() */ /*------------------------------------------------------------------------- * Function: H5D_istore_debug_key * * Purpose: Prints a key. * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * Thursday, April 16, 1998 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static herr_t H5D_istore_debug_key(FILE *stream, H5F_t UNUSED *f, hid_t UNUSED dxpl_id, int indent, int fwidth, const void *_key, const void *_udata) { const H5D_istore_key_t *key = (const H5D_istore_key_t *)_key; const H5D_istore_ud0_t *udata = (const H5D_istore_ud0_t *)_udata; unsigned u; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_debug_key) HDassert(key); HDfprintf(stream, "%*s%-*s %Zd bytes\n", indent, "", fwidth, "Chunk size:", key->nbytes); HDfprintf(stream, "%*s%-*s 0x%08x\n", indent, "", fwidth, "Filter mask:", key->filter_mask); HDfprintf(stream, "%*s%-*s {", indent, "", fwidth, "Logical offset:"); for(u = 0; u < udata->mesg->u.chunk.ndims; u++) HDfprintf(stream, "%s%Hd", u?", ":"", key->offset[u]); HDfputs("}\n", stream); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D_istore_debug_key() */ /*------------------------------------------------------------------------- * Function: H5D_istore_cmp2 * * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer * is only to supply extra information not carried in the keys * (in this case, the dimensionality) and is not compared * against the keys. * * Return: Success: -1 if LT_KEY is less than RT_KEY; * 1 if LT_KEY is greater than RT_KEY; * 0 if LT_KEY and RT_KEY are equal. * * Failure: FAIL (same as LT_KEY<RT_KEY) * * Programmer: Robb Matzke * Thursday, November 6, 1997 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int H5D_istore_cmp2(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, void *_lt_key, void *_udata, void *_rt_key) { H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; H5D_istore_ud0_t *udata = (H5D_istore_ud0_t *) _udata; int ret_value; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_cmp2) HDassert(lt_key); HDassert(rt_key); HDassert(udata); HDassert(udata->mesg->u.chunk.ndims > 0 && udata->mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); /* Compare the offsets but ignore the other fields */ ret_value = H5V_vector_cmp_u(udata->mesg->u.chunk.ndims, lt_key->offset, rt_key->offset); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_cmp2() */ /*------------------------------------------------------------------------- * Function: H5D_istore_cmp3 * * Purpose: Compare the requested datum UDATA with the left and right * keys of the B-tree. * * Return: Success: negative if the min_corner of UDATA is less * than the min_corner of LT_KEY. * * positive if the min_corner of UDATA is * greater than or equal the min_corner of * RT_KEY. * * zero otherwise. The min_corner of UDATA is * not necessarily contained within the address * space represented by LT_KEY, but a key that * would describe the UDATA min_corner address * would fall lexicographically between LT_KEY * and RT_KEY. * * Failure: FAIL (same as UDATA < LT_KEY) * * Programmer: Robb Matzke * Wednesday, October 8, 1997 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int H5D_istore_cmp3(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, void *_lt_key, void *_udata, void *_rt_key) { H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; H5D_istore_ud0_t *udata = (H5D_istore_ud0_t *) _udata; int ret_value = 0; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_cmp3) HDassert(lt_key); HDassert(rt_key); HDassert(udata); HDassert(udata->mesg->u.chunk.ndims > 0 && udata->mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); /* Special case for faster checks on 1-D chunks */ /* (Checking for ndims==2 because last dimension is the datatype size) */ /* The additional checking for the right key is necessary due to the */ /* slightly odd way the library initializes the right-most node in the */ /* indexed storage B-tree... */ /* (Dump the B-tree with h5debug to look at it) -QAK */ if(udata->mesg->u.chunk.ndims==2) { if(udata->key.offset[0]>rt_key->offset[0]) ret_value=1; else if(udata->key.offset[0]==rt_key->offset[0] && udata->key.offset[1]>=rt_key->offset[1]) ret_value=1; else if(udata->key.offset[0]<lt_key->offset[0]) ret_value=(-1); } /* end if */ else { if (H5V_vector_ge_u(udata->mesg->u.chunk.ndims, udata->key.offset, rt_key->offset)) ret_value = 1; else if (H5V_vector_lt_u(udata->mesg->u.chunk.ndims, udata->key.offset, lt_key->offset)) ret_value = -1; } /* end else */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_cmp3() */ /*------------------------------------------------------------------------- * Function: H5D_istore_new_node * * Purpose: Adds a new entry to an i-storage B-tree. We can assume that * the domain represented by UDATA doesn't intersect the domain * already represented by the B-tree. * * Return: Success: Non-negative. The address of leaf is returned * through the ADDR argument. It is also added * to the UDATA. * * Failure: Negative * * Programmer: Robb Matzke * Tuesday, October 14, 1997 * *------------------------------------------------------------------------- */ static herr_t H5D_istore_new_node(H5F_t *f, hid_t dxpl_id, H5B_ins_t op, void *_lt_key, void *_udata, void *_rt_key, haddr_t *addr_p/*out*/) { H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; unsigned u; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_new_node) /* check args */ HDassert(f); HDassert(lt_key); HDassert(rt_key); HDassert(udata); HDassert(udata->common.mesg->u.chunk.ndims > 0 && udata->common.mesg->u.chunk.ndims < H5O_LAYOUT_NDIMS); HDassert(addr_p); /* Allocate new storage */ HDassert(udata->common.key.nbytes > 0); H5_CHECK_OVERFLOW(udata->common.key.nbytes ,size_t, hsize_t); if(HADDR_UNDEF == (*addr_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->common.key.nbytes))) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "couldn't allocate new file storage") udata->addr = *addr_p; /* * The left key describes the storage of the UDATA chunk being * inserted into the tree. */ lt_key->nbytes = udata->common.key.nbytes; lt_key->filter_mask = udata->common.key.filter_mask; for (u=0; u<udata->common.mesg->u.chunk.ndims; u++) lt_key->offset[u] = udata->common.key.offset[u]; /* * The right key might already be present. If not, then add a zero-width * chunk. */ if (H5B_INS_LEFT != op) { rt_key->nbytes = 0; rt_key->filter_mask = 0; for (u=0; u<udata->common.mesg->u.chunk.ndims; u++) { HDassert(udata->common.key.offset[u]+udata->common.mesg->u.chunk.dim[u] > udata->common.key.offset[u]); rt_key->offset[u] = udata->common.key.offset[u] + udata->common.mesg->u.chunk.dim[u]; } } done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_new_node() */ /*------------------------------------------------------------------------- * Function: H5D_istore_found * * Purpose: This function is called when the B-tree search engine has * found the leaf entry that points to a chunk of storage that * contains the beginning of the logical address space * represented by UDATA. The LT_KEY is the left key (the one * that describes the chunk) and RT_KEY is the right key (the * one that describes the next or last chunk). * * Note: It's possible that the chunk isn't really found. For * instance, in a sparse dataset the requested chunk might fall * between two stored chunks in which case this function is * called with the maximum stored chunk indices less than the * requested chunk indices. * * Return: Non-negative on success with information about the chunk * returned through the UDATA argument. Negative on failure. * * Programmer: Robb Matzke * Thursday, October 9, 1997 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static herr_t H5D_istore_found(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, haddr_t addr, const void *_lt_key, void *_udata) { H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *) _lt_key; unsigned u; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_found) /* Check arguments */ HDassert(f); HDassert(H5F_addr_defined(addr)); HDassert(udata); HDassert(lt_key); /* Is this *really* the requested chunk? */ for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) if(udata->common.key.offset[u] >= lt_key->offset[u] + udata->common.mesg->u.chunk.dim[u]) HGOTO_DONE(FAIL) /* Initialize return values */ udata->addr = addr; udata->common.key.nbytes = lt_key->nbytes; udata->common.key.filter_mask = lt_key->filter_mask; HDassert(lt_key->nbytes>0); for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) udata->common.key.offset[u] = lt_key->offset[u]; done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_found() */ /*------------------------------------------------------------------------- * Function: H5D_istore_insert * * Purpose: This function is called when the B-tree insert engine finds * the node to use to insert new data. The UDATA argument * points to a struct that describes the logical addresses being * added to the file. This function allocates space for the * data and returns information through UDATA describing a * file chunk to receive (part of) the data. * * The LT_KEY is always the key describing the chunk of file * memory at address ADDR. On entry, UDATA describes the logical * addresses for which storage is being requested (through the * `offset' and `size' fields). On return, UDATA describes the * logical addresses contained in a chunk on disk. * * Return: Success: An insertion command for the caller, one of * the H5B_INS_* constants. The address of the * new chunk is returned through the NEW_NODE * argument. * * Failure: H5B_INS_ERROR * * Programmer: Robb Matzke * Thursday, October 9, 1997 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static H5B_ins_t H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, hbool_t *lt_key_changed, void *_md_key, void *_udata, void *_rt_key, hbool_t UNUSED *rt_key_changed, haddr_t *new_node_p/*out*/) { H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *md_key = (H5D_istore_key_t *) _md_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; int cmp; unsigned u; H5B_ins_t ret_value; FUNC_ENTER_NOAPI_NOINIT(H5D_istore_insert) /* check args */ HDassert(f); HDassert(H5F_addr_defined(addr)); HDassert(lt_key); HDassert(lt_key_changed); HDassert(md_key); HDassert(udata); HDassert(rt_key); HDassert(new_node_p); cmp = H5D_istore_cmp3(f, dxpl_id, lt_key, udata, rt_key); HDassert(cmp <= 0); if (cmp < 0) { /* Negative indices not supported yet */ HGOTO_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, H5B_INS_ERROR, "internal error") } else if (H5V_vector_eq_u (udata->common.mesg->u.chunk.ndims, udata->common.key.offset, lt_key->offset) && lt_key->nbytes>0) { /* * Already exists. If the new size is not the same as the old size * then we should reallocate storage. */ if (lt_key->nbytes != udata->common.key.nbytes) { /* Currently, the old chunk data is "thrown away" after the space is reallocated, * so avoid data copy in H5MF_realloc() call by just free'ing the space and * allocating new space. * * This should keep the file smaller also, by freeing the space and then * allocating new space, instead of vice versa (in H5MF_realloc). * * QAK - 11/19/2002 */ #ifdef OLD_WAY if(HADDR_UNDEF == (*new_node_p = H5MF_realloc(f, H5FD_MEM_DRAW, addr, (hsize_t)lt_key->nbytes, (hsize_t)udata->common.key.nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk storage") #else /* OLD_WAY */ H5_CHECK_OVERFLOW( lt_key->nbytes ,size_t, hsize_t); if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk") H5_CHECK_OVERFLOW(udata->common.key.nbytes ,size_t, hsize_t); if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->common.key.nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk") #endif /* OLD_WAY */ lt_key->nbytes = udata->common.key.nbytes; lt_key->filter_mask = udata->common.key.filter_mask; *lt_key_changed = TRUE; udata->addr = *new_node_p; ret_value = H5B_INS_CHANGE; } else { udata->addr = addr; ret_value = H5B_INS_NOOP; } } else if (H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, lt_key->offset, udata->common.mesg->u.chunk.dim, udata->common.key.offset, udata->common.mesg->u.chunk.dim)) { HDassert(H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, rt_key->offset, udata->common.mesg->u.chunk.dim, udata->common.key.offset, udata->common.mesg->u.chunk.dim)); /* * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. */ md_key->nbytes = udata->common.key.nbytes; md_key->filter_mask = udata->common.key.filter_mask; for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) { HDassert(0 == udata->common.key.offset[u] % udata->common.mesg->u.chunk.dim[u]); md_key->offset[u] = udata->common.key.offset[u]; } /* * Allocate storage for the new chunk */ H5_CHECK_OVERFLOW(udata->common.key.nbytes ,size_t, hsize_t); if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->common.key.nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "file allocation failed") udata->addr = *new_node_p; ret_value = H5B_INS_RIGHT; } else { HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, H5B_INS_ERROR, "internal error") } done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_insert() */ /*------------------------------------------------------------------------- * Function: H5D_istore_iter_allocated * * Purpose: Simply counts the number of chunks for a dataset. * * Return: Success: Non-negative * * Failure: Negative * * Programmer: Robb Matzke * Wednesday, April 21, 1999 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int H5D_istore_iter_allocated (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t UNUSED addr, const void UNUSED *_rt_key, void *_udata) { H5D_istore_it_ud1_t *udata = (H5D_istore_it_ud1_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_iter_allocated) udata->total_storage += lt_key->nbytes; FUNC_LEAVE_NOAPI(H5B_ITER_CONT) } /* H5D_istore_iter_allocated() */ /*------------------------------------------------------------------------- * Function: H5D_istore_iter_chunkmap * * Purpose: obtain chunk address and the corresponding index * * Return: Success: Non-negative * * Failure: Negative * * Programmer: Kent Yang * Tuesday, November 15, 2005 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int H5D_istore_iter_chunkmap (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t addr, const void UNUSED *_rt_key, void *_udata) { H5D_istore_it_ud5_t *udata = (H5D_istore_it_ud5_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; unsigned rank; hsize_t chunk_index; int ret_value = H5B_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_chunkmap); rank = udata->common.mesg->u.chunk.ndims - 1; if(H5V_chunk_index(rank,lt_key->offset,udata->common.mesg->u.chunk.dim,udata->down_chunks,&chunk_index)<0) HGOTO_ERROR (H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") udata->chunk_addr[chunk_index] = addr; done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D_istore_iter_allocated() */ /*------------------------------------------------------------------------- * Function: H5D_istore_iter_dump * * Purpose: If the UDATA.STREAM member is non-null then debugging * information is written to that stream. * * Return: Success: Non-negative * * Failure: Negative * * Programmer: Robb Matzke * Wednesday, April 21, 1999 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int H5D_istore_iter_dump (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t UNUSED addr, const void UNUSED *_rt_key, void *_udata) { H5D_istore_it_ud2_t *udata = (H5D_istore_it_ud2_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; unsigned u; FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_iter_dump) if(udata->stream) { if(!udata->header_displayed) { HDfprintf(udata->stream, " Flags Bytes Address Logical Offset\n"); HDfprintf(udata->stream, " ========== ======== ========== ==============================\n"); /* Set flag that the headers has been printed */ udata->header_displayed = TRUE; } /* end if */ HDfprintf(udata->stream, " 0x%08x %8Zu %10a [", lt_key->filter_mask, lt_key->nbytes, addr); for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) HDfprintf(udata->stream, "%s%Hd", (u ? ", " : ""), lt_key->offset[u]); HDfputs("]\n", udata->stream); } /* end if */ FUNC_LEAVE_NOAPI(H5B_ITER_CONT) } /* H5D_istore_iter_dump() */ /*------------------------------------------------------------------------- * Function: H5D_istore_iter_copy * * Purpose: copy chunked raw data from source file and insert to the * B-tree node in the destination file * * Return: Non-negative on success/Negative on failure * * Programmer: Peter Cao * August 20, 2005 * *------------------------------------------------------------------------- */ static int H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, haddr_t addr_src, const void UNUSED *_rt_key, void *_udata) { H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ void *bkg = NULL; /* Temporary buffer for copying data */ hbool_t is_vlen = FALSE; /* General information about chunk copy */ void *buf = udata->buf; size_t buf_size = udata->buf_size; H5O_pline_t *pline = udata->pline; /* needed for commpressed variable length data */ hbool_t is_compressed = FALSE; H5Z_EDC_t edc_read = H5Z_NO_EDC; size_t nbytes = lt_key->nbytes; H5Z_cb_t cb_struct; int ret_value = H5B_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy) /* Check parameter for type conversion */ if (udata->tid_src > 0) is_vlen = TRUE; /* Check for filtered chunks */ if (pline && pline->nused) { is_compressed = TRUE; cb_struct.func = NULL; /* no callback function when failed */ } /* end if */ /* Resize the buf if it is too small to hold the data */ if ( nbytes > buf_size) { /* Re-allocate memory for copying the chunk */ if(NULL == (udata->buf = H5MM_realloc(udata->buf, nbytes))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed for raw data chunk") buf = udata->buf; udata->buf_size = buf_size = nbytes; } /* end if */ /* read chunk data from the source file */ if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, H5B_ITER_ERROR, "unable to read raw data chunk") /* need to uncompress variable-length data */ if (is_compressed && is_vlen) { unsigned filter_mask = lt_key->filter_mask; if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "data pipeline read failed") } /* end if */ /* Perform datatype conversion, if necessary */ if(is_vlen) { H5T_path_t *tpath_src_mem = udata->tpath_src_mem; H5T_path_t *tpath_mem_dst = udata->tpath_mem_dst; H5S_t *buf_space = udata->buf_space; hid_t tid_src = udata->tid_src; hid_t tid_dst = udata->tid_dst; hid_t tid_mem = udata->tid_mem; size_t nelmts = udata->nelmts; void *reclaim_buf = udata->reclaim_buf; size_t reclaim_buf_size = udata->reclaim_buf_size; /* Convert from source file to memory */ if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") /* Copy into another buffer, to reclaim memory later */ HDmemcpy(reclaim_buf, buf, reclaim_buf_size); /* allocate temporary bkg buff for data conversion */ if(NULL == (bkg = H5FL_BLK_CALLOC(type_conv, buf_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed") /* Convert from memory to destination file */ if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") /* Reclaim space from variable length data */ if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_BADITER, H5B_ITER_ERROR, "unable to reclaim variable-length data") } /* end if */ /* Copy source chunk callback information for insertion */ HDmemset(&udata_dst, 0, sizeof(udata_dst)); HDmemcpy(&(udata_dst.common.key), lt_key, sizeof(H5D_istore_key_t)); udata_dst.common.mesg = udata->common.mesg; /* Share this pointer for a short while */ /* need to compress variable-length data before writing to file*/ if (is_compressed && is_vlen) { if(H5Z_pipeline(pline, 0, &(udata_dst.common.key.filter_mask), edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "output pipeline failed") udata_dst.common.key.nbytes = nbytes; udata->buf = buf; udata->buf_size = buf_size; } /* end if */ /* Insert chunk into the destination Btree */ if(H5B_insert(udata->file_dst, dxpl_id, H5B_ISTORE, udata->addr_dst, &udata_dst) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to allocate chunk") /* Write chunk data to destination file */ HDassert(H5F_addr_defined(udata_dst.addr)); if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to write raw data to file") done: if(bkg) H5FL_BLK_FREE(type_conv, bkg); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_iter_copy() */ /*------------------------------------------------------------------------- * Function: H5D_istore_init * * Purpose: Initialize the raw data chunk cache for a dataset. This is * called when the dataset is initialized. * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * Monday, May 18, 1998 * *------------------------------------------------------------------------- */ herr_t H5D_istore_init (const H5F_t *f, const H5D_t *dset) { H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_istore_init, FAIL) if (H5F_RDCC_NBYTES(f)>0 && H5F_RDCC_NELMTS(f)>0) { rdcc->nbytes=H5F_RDCC_NBYTES(f); rdcc->nslots = H5F_RDCC_NELMTS(f); rdcc->slot = H5FL_SEQ_CALLOC (H5D_rdcc_ent_ptr_t,rdcc->nslots); if (NULL==rdcc->slot) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") } /* end if */ /* Allocate the shared structure */ if(H5D_istore_shared_create(f, &dset->shared->layout)<0) HGOTO_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_init() */ /*------------------------------------------------------------------------- * Function: H5D_istore_flush_entry * * Purpose: Writes a chunk to disk. If RESET is non-zero then the * entry is cleared -- it's slightly faster to flush a chunk if * the RESET flag is turned on because it results in one fewer * memory copy. * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * Thursday, May 21, 1998 * *------------------------------------------------------------------------- */ static herr_t H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_t reset) { herr_t ret_value=SUCCEED; /*return value */ unsigned u; /*counters */ void *buf=NULL; /*temporary buffer */ size_t alloc; /*bytes allocated for BUF */ hbool_t point_of_no_return = FALSE; FUNC_ENTER_NOAPI_NOINIT(H5D_istore_flush_entry) assert(io_info); assert(io_info->dset); assert(ent); assert(!ent->locked); buf = ent->chunk; if (ent->dirty) { H5D_istore_ud1_t udata; /*pass through B-tree */ udata.common.mesg = &io_info->dset->shared->layout; udata.common.key.filter_mask = 0; udata.addr = HADDR_UNDEF; udata.common.key.nbytes = ent->chunk_size; for (u=0; u<io_info->dset->shared->layout.u.chunk.ndims; u++) udata.common.key.offset[u] = ent->offset[u]; alloc = ent->alloc_size; /* Should the chunk be filtered before writing it to disk? */ if (io_info->dset->shared->dcpl_cache.pline.nused) { if (!reset) { /* * Copy the chunk to a new buffer before running it through * the pipeline because we'll want to save the original buffer * for later. */ alloc = ent->chunk_size; if (NULL==(buf = H5MM_malloc(alloc))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline") HDmemcpy(buf, ent->chunk, ent->chunk_size); } else { /* * If we are reseting and something goes wrong after this * point then it's too late to recover because we may have * destroyed the original data by calling H5Z_pipeline(). * The only safe option is to continue with the reset * even if we can't write the data to disk. */ point_of_no_return = TRUE; ent->chunk = NULL; } if (H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.common.key.filter_mask), io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb, &(udata.common.key.nbytes), &alloc, &buf)<0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") } /* * Create the chunk it if it doesn't exist, or reallocate the chunk if * its size changed. Then write the data into the file. */


context:
space:
mode: