[svn-r6825] Purpose:

    New feature/enhancement

Description:
    Chunked datasets are handled poorly in several circumstances involving
certain selections and chunks that are too large for the chunk cache and/or
chunks with filters, causing the chunk to be read from disk multiple times.

Solution:
    Rearrange raw data I/O infrastructure to handle chunked datasets in a much
more friendly way by creating a selection in memory and on disk for each chunk
in a chunked dataset and performing all of the I/O on that chunk at one time.

    There are still some scalability (the current code attempts to
create a selection for all the chunks in the dataset, instead of just the
chunks that are accessed, requiring portions of the istore.c and fillval.c
tests to be commented out) and performance issues, but checking this in will
allow the changes to be tested by a much wider audience while I address the
remaining issues.


Platforms tested:
    h5committested, FreeBSD 4.8 (sleipnir) serial & parallel, Linux 2.4 (eirene)

1 files changed, 59 insertions, 19 deletions

diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index e6619a4..6ca9802 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -51,17 +51,16 @@ typedef struct H5S_pnt_node_t H5S_pnt_node_t;
 typedef struct H5S_hyper_span_t H5S_hyper_span_t;
 typedef struct H5S_hyper_span_info_t H5S_hyper_span_info_t;
 typedef struct H5S_hyper_dim_t H5S_hyper_dim_t;
+union H5D_storage_t;
 
 /* Point selection iteration container */
 typedef struct {
-    hsize_t elmt_left;      /* Number of elements left to iterate over */
     H5S_pnt_node_t *curr;   /* Pointer to next node to output */
 } H5S_point_iter_t;
 
-/* New Hyperslab selection iteration container */
+/* Hyperslab selection iteration container */
 typedef struct {
     /* Common fields for all hyperslab selections */
-    hsize_t elmt_left;      /* Number of elements left to iterate over */
     hssize_t *off;          /* Offset in span node (used as position for regular hyperslabs) */
     unsigned iter_rank;     /* Rank of iterator information */
                             /* (This should always be the same as the dataspace
@@ -82,16 +81,40 @@ typedef struct {
 
 /* "All" selection iteration container */
 typedef struct {
-    hsize_t elmt_left;      /* Number of elements left to iterate over */
     hsize_t offset;         /* Next element to output */
 } H5S_all_iter_t;
 
+typedef struct H5S_sel_iter_t H5S_sel_iter_t;
+
+/* Method to retrieve the current coordinates of iterator for current selection */
+typedef herr_t (*H5S_sel_iter_coords_func_t)(const H5S_sel_iter_t *iter, hssize_t *coords);
+/* Method to determine number of elements left in iterator for current selection */
+typedef hsize_t (*H5S_sel_iter_nelmts_func_t)(const H5S_sel_iter_t *iter);
+/* Method to move selection iterator to the next element in the selection */
+typedef herr_t (*H5S_sel_iter_next_func_t)(H5S_sel_iter_t *iter, size_t nelem);
+/* Method to release iterator for current selection */
+typedef herr_t (*H5S_sel_iter_release_func_t)(H5S_sel_iter_t *iter);
+
 /* Selection iteration container */
-typedef union {
-    H5S_point_iter_t pnt;   /* Point selection iteration information */
-    H5S_hyper_iter_t hyp;   /* New Hyperslab selection iteration information */
-    H5S_all_iter_t all;     /* "All" selection iteration information */
-} H5S_sel_iter_t;
+struct H5S_sel_iter_t {
+    /* Information common to all iterators */
+    unsigned rank;              /* Rank of dataspace the selection iterator is operating on */
+    hsize_t *dims;              /* Dimensions of dataspace the selection is operating on */
+    hsize_t elmt_left;          /* Number of elements left to iterate over */
+
+    /* Information specific to each type of iterator */
+    union {
+        H5S_point_iter_t pnt;   /* Point selection iteration information */
+        H5S_hyper_iter_t hyp;   /* New Hyperslab selection iteration information */
+        H5S_all_iter_t all;     /* "All" selection iteration information */
+    } u;
+
+    /* Methods on selections */
+    H5S_sel_iter_coords_func_t iter_coords;     /* Method to retrieve the current coordinates of iterator for current selection */
+    H5S_sel_iter_nelmts_func_t iter_nelmts;     /* Method to determine number of elements left in iterator for current selection */
+    H5S_sel_iter_next_func_t iter_next;         /* Method to move selection iterator to the next element in the selection */
+    H5S_sel_iter_release_func_t iter_release;   /* Method to release iterator for current selection */
+};
 
 typedef struct H5S_conv_t {
     H5S_sel_type	ftype;
@@ -105,14 +128,14 @@ typedef struct H5S_conv_t {
     
     /* Read from file to application w/o intermediate scratch buffer */
     herr_t (*read)(H5F_t *f, const struct H5O_layout_t *layout,
-           H5P_genplist_t *dc_plist, const H5O_efl_t *efl,
+           H5P_genplist_t *dc_plist, const union H5D_storage_t *store,
            size_t elmt_size, const H5S_t *file_space,
            const H5S_t *mem_space, hid_t dxpl_id, void *buf/*out*/);
 
 
     /* Write directly from app buffer to file */
     herr_t (*write)(H5F_t *f, struct H5O_layout_t *layout,
-           H5P_genplist_t *dc_plist, const H5O_efl_t *efl,
+           H5P_genplist_t *dc_plist, const union H5D_storage_t *store, 
            size_t elmt_size, const H5S_t *file_space,
            const H5S_t *mem_space, hid_t dxpl_id, const void *buf);
     
@@ -161,7 +184,7 @@ H5_DLL htri_t H5S_is_simple(const H5S_t *sdim);
 H5_DLL herr_t H5S_extent_release(H5S_t *space);
 H5_DLL int H5S_extend(H5S_t *space, const hsize_t *size);
 H5_DLL int H5S_set_extent(H5S_t *space, const hsize_t *size);
-H5_DLL H5S_t *H5S_create_simple(int rank, const hsize_t dims[/*rank*/],
+H5_DLL H5S_t *H5S_create_simple(unsigned rank, const hsize_t dims[/*rank*/],
 		  const hsize_t maxdims[/*rank*/]);
 H5_DLL herr_t H5S_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE *stream,
 			 int indent, int fwidth);
@@ -169,17 +192,18 @@ H5_DLL herr_t H5S_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE *stream
 /* Operations on selections */
 H5_DLL herr_t H5S_select_copy(H5S_t *dst, const H5S_t *src);
 H5_DLL herr_t H5S_select_deserialize(H5S_t *space, const uint8_t *buf);
+H5_DLL H5S_sel_type H5S_get_select_type(const H5S_t *space);
 H5_DLL htri_t H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2);
-H5_DLL herr_t H5S_select_iterate(void *buf, hid_t type_id, H5S_t *space,
+H5_DLL herr_t H5S_select_iterate(void *buf, hid_t type_id, const H5S_t *space,
 				H5D_operator_t op, void *operator_data);
 H5_DLL herr_t H5S_select_fill(void *fill, size_t fill_size,
                                 const H5S_t *space, void *buf);
 H5_DLL herr_t H5S_select_fscat (H5F_t *f, struct H5O_layout_t *layout,
-        H5P_genplist_t *dc_plist, const H5O_efl_t *efl, size_t elmt_size,
+        H5P_genplist_t *dc_plist, const union H5D_storage_t *store, size_t elmt_size,
         const H5S_t *file_space, H5S_sel_iter_t *file_iter, hsize_t nelmts,
         hid_t dxpl_id, const void *_buf);
 H5_DLL hsize_t H5S_select_fgath (H5F_t *f, const struct H5O_layout_t *layout,
-        H5P_genplist_t *dc_plist, const H5O_efl_t *efl, size_t elmt_size,
+        H5P_genplist_t *dc_plist, const union H5D_storage_t *store, size_t elmt_size,
         const H5S_t *file_space, H5S_sel_iter_t *file_iter, hsize_t nelmts,
         hid_t dxpl_id, void *buf);
 H5_DLL herr_t H5S_select_mscat (const void *_tscat_buf, size_t elmt_size,
@@ -189,19 +213,35 @@ H5_DLL hsize_t H5S_select_mgath (const void *_buf, size_t elmt_size,
         const H5S_t *space, H5S_sel_iter_t *iter, hsize_t nelmts,
         hid_t dxpl_id, void *_tgath_buf/*out*/);
 H5_DLL herr_t H5S_select_read(H5F_t *f, const struct H5O_layout_t *layout,
-        H5P_genplist_t *dc_plist, const H5O_efl_t *efl, size_t elmt_size,
+        H5P_genplist_t *dc_plist, const union H5D_storage_t *store, size_t elmt_size,
         const H5S_t *file_space, const H5S_t *mem_space, hid_t dxpl_id,
         void *buf/*out*/);
 H5_DLL herr_t H5S_select_write(H5F_t *f, struct H5O_layout_t *layout,
-        H5P_genplist_t *dc_plist, const H5O_efl_t *efl, size_t elmt_size,
+        H5P_genplist_t *dc_plist, const union H5D_storage_t *store, size_t elmt_size,
         const H5S_t *file_space, const H5S_t *mem_space, hid_t dxpl_id,
         const void *buf/*out*/);
-
-/* Needed for internal use of selections in H5Fistore code */
+H5_DLL herr_t H5S_select_elements (H5S_t *space, H5S_seloper_t op, 
+        size_t num_elem, const hssize_t **coord);
+#ifdef NEW_HYPERSLAB_API
+H5_DLL herr_t H5S_select_select (H5S_t *space1, H5S_seloper_t op, H5S_t *space2);
+#endif /*NEW_HYPERSLAB_API */
+H5_DLL htri_t H5S_select_valid(const H5S_t *space);
+H5_DLL hssize_t H5S_get_select_npoints(const H5S_t *space);
 H5_DLL herr_t H5S_select_all(H5S_t *space, unsigned rel_prev);
+H5_DLL herr_t H5S_select_none(H5S_t *space);
 H5_DLL herr_t H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hssize_t start[],
                 const hsize_t *stride, const hsize_t count[],
                 const hsize_t *block);
+H5_DLL herr_t H5S_hyper_add_span_element(H5S_t *space, unsigned rank,
+    hssize_t *coords);
+H5_DLL herr_t H5S_hyper_reset_scratch(H5S_t *space);
+
+/* Operations on selection iterators */
+H5_DLL herr_t H5S_select_iter_init(H5S_sel_iter_t *iter, const H5S_t *space, size_t elmt_size);
+H5_DLL herr_t H5S_select_iter_coords(const H5S_sel_iter_t *sel_iter, hssize_t *coords);
+H5_DLL hsize_t H5S_select_iter_nelmts(const H5S_sel_iter_t *sel_iter);
+H5_DLL herr_t H5S_select_iter_next(H5S_sel_iter_t *sel_iter, size_t nelem);
+H5_DLL herr_t H5S_select_iter_release(H5S_sel_iter_t *sel_iter);
 
 #ifdef H5_HAVE_PARALLEL
 /* MPI-IO function to read directly from app buffer to file rky980813 */