summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorScot Breitenfeld <brtnfld@hdfgroup.org>2017-10-26 19:29:27 (GMT)
committerScot Breitenfeld <brtnfld@hdfgroup.org>2017-10-26 19:29:27 (GMT)
commit85531beb2afc266b203eeeeb205f7bc95cdcc9b2 (patch)
tree8582ae2861dd0ef9b1df37befc9ba9ca09cd6f34 /src
parentc222f391c43f91638368e4a2e8c7b556fa5b2fbb (diff)
parent2074cbd516d3dcc6aa6a3f8aa2978017baf8c5cc (diff)
downloadhdf5-85531beb2afc266b203eeeeb205f7bc95cdcc9b2.zip
hdf5-85531beb2afc266b203eeeeb205f7bc95cdcc9b2.tar.gz
hdf5-85531beb2afc266b203eeeeb205f7bc95cdcc9b2.tar.bz2
Merge pull request #730 in HDFFV/hdf5 from ~BRTNFLD/hdf5_msb:hdf5_1_10 to hdf5_1_10
* commit '2074cbd516d3dcc6aa6a3f8aa2978017baf8c5cc': (397 commits) fixed merge with develop issues HDFFV-10037: fixed wrong C link flags Correct typo fix typo Fix typos HDFFV-10297 Free buffer inside loop HDFFV-10297 Cleanup, Initialize variables Moved the SWMR + cache image check up before the root group is constructed to avoid the special case close. HDFFV-10297 Windows issues fixed Windows cannot share files easily Moved the 'cache image + SWMR' check from H5Fcreate/open to H5F_open. Prep for the VOL merge. Avoid double free Windows had issues - revert code changes for get_option Remove extra command line Correct name of file Add Mask test to script Add new output files to clear test Correct name of err file Fix name of output files Fix format convert error mask test ...
Diffstat (limited to 'src')
-rw-r--r--src/CMakeLists.txt3
-rw-r--r--src/H5.c7
-rw-r--r--src/H5ACprivate.h3
-rw-r--r--src/H5C.c81
-rw-r--r--src/H5Cdbg.c2
-rw-r--r--src/H5Cimage.c2
-rw-r--r--src/H5Cpkg.h2
-rw-r--r--src/H5Cprivate.h2
-rw-r--r--src/H5Dchunk.c5
-rw-r--r--src/H5Defl.c4
-rw-r--r--src/H5Dint.c4
-rw-r--r--src/H5Dio.c68
-rw-r--r--src/H5Dmpio.c1665
-rw-r--r--src/H5Dpkg.h8
-rw-r--r--src/H5Dscatgath.c7
-rw-r--r--src/H5F.c22
-rw-r--r--src/H5FDcore.c10
-rw-r--r--src/H5FDdirect.c2
-rw-r--r--src/H5FDlog.c2
-rw-r--r--src/H5FDmpi.c5
-rw-r--r--src/H5FDsec2.c2
-rw-r--r--src/H5FSint.c26
-rw-r--r--src/H5FSprivate.h3
-rw-r--r--src/H5Fint.c17
-rw-r--r--src/H5Fpkg.h30
-rw-r--r--src/H5Fsuper.c136
-rw-r--r--src/H5I.c4
-rw-r--r--src/H5Ipublic.h9
-rw-r--r--src/H5MF.c1
-rw-r--r--src/H5MFdbg.c2
-rw-r--r--src/H5Oefl.c2
-rw-r--r--src/H5Oflush.c38
-rw-r--r--src/H5Otest.c2
-rw-r--r--src/H5PB.c4
-rw-r--r--src/H5PL.c988
-rw-r--r--src/H5PLint.c384
-rw-r--r--src/H5PLpath.c776
-rw-r--r--src/H5PLpkg.h113
-rw-r--r--src/H5PLplugin_cache.c307
-rw-r--r--src/H5PLpublic.h34
-rw-r--r--src/H5Ppublic.h2
-rw-r--r--src/H5R.c10
-rw-r--r--src/H5err.txt2
-rw-r--r--src/H5private.h15
-rw-r--r--src/H5system.c46
-rw-r--r--src/H5trace.c4
-rw-r--r--src/H5win32defs.h9
-rw-r--r--src/Makefile.am2
48 files changed, 3831 insertions, 1041 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 178c954..96ea589 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" )
set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
+ ${HDF5_SRC_DIR}/H5PLint.c
+ ${HDF5_SRC_DIR}/H5PLpath.c
+ ${HDF5_SRC_DIR}/H5PLplugin_cache.c
)
set (H5PL_HDRS
diff --git a/src/H5.c b/src/H5.c
index 1068fc6..59984dc 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -29,6 +29,7 @@
#include "H5Pprivate.h" /* Property lists */
#include "H5SLprivate.h" /* Skip lists */
#include "H5Tprivate.h" /* Datatypes */
+#include "H5FSprivate.h" /* File free space */
/****************/
/* Local Macros */
@@ -204,6 +205,10 @@ H5_init_library(void)
* property classes.
* The link interface needs to be initialized so that link property lists
* have their properties registered.
+ * The FS module needs to be initialized as a result of the fix for HDFFV-10160:
+ * It might not be initialized during normal file open.
+ * When the application does not close the file, routines in the module might
+ * be called via H5_term_library() when shutting down the file.
*/
if(H5E_init() < 0)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface")
@@ -217,6 +222,8 @@ H5_init_library(void)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize metadata caching interface")
if(H5L_init() < 0)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize link interface")
+ if(H5FS_init() < 0)
+ HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize FS interface")
/* Debugging? */
H5_debug_mask("-all");
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index b9e2a60..083ee5b 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -230,7 +230,7 @@ typedef struct H5AC_proxy_entry_t {
/* (Note that this currently duplicates some cache functionality) */
} H5AC_proxy_entry_t;
-
+/* Name of property for ring info in DXPL */
#define H5AC_RING_NAME "H5AC_ring_type"
/* Dataset transfer property lists for metadata calls */
@@ -247,7 +247,6 @@ H5_DLLVAR hid_t H5AC_noio_dxpl_id;
H5_DLLVAR hid_t H5AC_rawdata_dxpl_id;
/* Default cache configuration. */
-
#define H5AC__DEFAULT_METADATA_WRITE_STRATEGY \
H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED
diff --git a/src/H5C.c b/src/H5C.c
index e6770ec..0d8cc75 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -366,6 +366,7 @@ H5C_create(size_t max_cache_size,
cache_ptr->coll_write_list = NULL;
#endif /* H5_HAVE_PARALLEL */
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
cache_ptr->cLRU_list_len = 0;
cache_ptr->cLRU_list_size = (size_t)0;
cache_ptr->cLRU_head_ptr = NULL;
@@ -375,6 +376,7 @@ H5C_create(size_t max_cache_size,
cache_ptr->dLRU_list_size = (size_t)0;
cache_ptr->dLRU_head_ptr = NULL;
cache_ptr->dLRU_tail_ptr = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
cache_ptr->size_increase_possible = FALSE;
cache_ptr->flash_size_increase_possible = FALSE;
@@ -1481,8 +1483,10 @@ H5C_insert_entry(H5F_t * f,
entry_ptr->next = NULL;
entry_ptr->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
entry_ptr->aux_next = NULL;
entry_ptr->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
entry_ptr->coll_next = NULL;
@@ -4062,8 +4066,7 @@ H5C_destroy_flush_dependency(void *parent_thing, void * child_thing)
child_entry->flush_dep_parent_nalloc = 0;
} /* end if */
else if(child_entry->flush_dep_parent_nalloc > H5C_FLUSH_DEP_PARENT_INIT
- && child_entry->flush_dep_nparents
- <= (child_entry->flush_dep_parent_nalloc / 4)) {
+ && child_entry->flush_dep_nparents <= (child_entry->flush_dep_parent_nalloc / 4)) {
if(NULL == (child_entry->flush_dep_parent = (H5C_cache_entry_t **)H5FL_BLK_REALLOC(parent, child_entry->flush_dep_parent, (child_entry->flush_dep_parent_nalloc / 4) * sizeof(H5C_cache_entry_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for flush dependency parent list")
child_entry->flush_dep_parent_nalloc /= 4;
@@ -6273,17 +6276,27 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item")
} /* end if */
else
+ {
#endif /* H5_HAVE_PARALLEL */
- if(entry_ptr->prefetched) {
- HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID);
- mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type;
- } /* end if */
- else
- mem_type = entry_ptr->type->mem_type;
+ if(entry_ptr->prefetched) {
+ HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID);
+ mem_type = cache_ptr->
+ class_table_ptr[entry_ptr->prefetch_type_id]->
+ mem_type;
+ } /* end if */
+ else
+ mem_type = entry_ptr->type->mem_type;
+
+ if(H5F_block_write(f, mem_type, entry_ptr->addr,
+ entry_ptr->size, dxpl_id,
+ entry_ptr->image_ptr) < 0)
- if(H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+ "Can't write image to file")
+#ifdef H5_HAVE_PARALLEL
+ }
+#endif /* H5_HAVE_PARALLEL */
} /* end if */
/* if the entry has a notify callback, notify it that we have
@@ -6418,7 +6431,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
HDassert(entry_ptr->flush_dep_ndirty_children == 0);
if(entry_ptr->flush_dep_nparents > 0)
if(H5C__mark_flush_dep_clean(entry_ptr) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean flag")
} /* end if */
} /* end else */
@@ -6940,8 +6953,10 @@ H5C_load_entry(H5F_t * f,
entry->next = NULL;
entry->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
entry->aux_next = NULL;
entry->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
entry->coll_next = NULL;
@@ -7276,6 +7291,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
HDassert( H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS );
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
initial_list_len = cache_ptr->cLRU_list_len;
entry_ptr = cache_ptr->cLRU_tail_ptr;
@@ -7320,6 +7336,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
entry_ptr = prev_ptr;
entries_examined++;
}
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
}
done:
@@ -7979,7 +7996,7 @@ done:
static herr_t
H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry)
{
- unsigned u; /* Local index variable */
+ int i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -7988,16 +8005,19 @@ H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry)
HDassert(entry);
/* Iterate over the parent entries, if any */
- for(u = 0; u < entry->flush_dep_nparents; u++) {
+ /* Note reverse iteration order, in case the callback removes the flush
+ * dependency - QAK, 2017/08/12
+ */
+ for(i = ((int)entry->flush_dep_nparents) - 1; i >= 0; i--) {
/* Sanity check */
- HDassert(entry->flush_dep_parent[u]->flush_dep_ndirty_children > 0);
+ HDassert(entry->flush_dep_parent[i]->flush_dep_ndirty_children > 0);
/* Adjust the parent's number of dirty children */
- entry->flush_dep_parent[u]->flush_dep_ndirty_children--;
+ entry->flush_dep_parent[i]->flush_dep_ndirty_children--;
/* If the parent has a 'notify' callback, send a 'child entry cleaned' notice */
- if(entry->flush_dep_parent[u]->type->notify &&
- (entry->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[u]) < 0)
+ if(entry->flush_dep_parent[i]->type->notify &&
+ (entry->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[i]) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag reset")
} /* end for */
@@ -8023,7 +8043,7 @@ done:
herr_t
H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr)
{
- unsigned u; /* Local index variable */
+ int i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -8032,18 +8052,21 @@ H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr)
HDassert(entry_ptr);
/* Iterate over the parent entries, if any */
- for(u = 0; u < entry_ptr->flush_dep_nparents; u++) {
-
+ /* Note reverse iteration order, in case the callback removes the flush
+ * dependency - QAK, 2017/08/12
+ */
+ for(i = ((int)entry_ptr->flush_dep_nparents) - 1; i >= 0; i--) {
+ /* Sanity checks */
HDassert(entry_ptr->flush_dep_parent);
- HDassert(entry_ptr->flush_dep_parent[u]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
- HDassert(entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children > 0);
+ HDassert(entry_ptr->flush_dep_parent[i]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+ HDassert(entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children > 0);
/* decrement the parents number of unserialized children */
- entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children--;
+ entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children--;
/* If the parent has a 'notify' callback, send a 'child entry serialized' notice */
- if(entry_ptr->flush_dep_parent[u]->type->notify &&
- (entry_ptr->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[u]) < 0)
+ if(entry_ptr->flush_dep_parent[i]->type->notify &&
+ (entry_ptr->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[i]) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry serialized flag set")
} /* end for */
@@ -8055,7 +8078,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5C__mark_flush_dep_unserialized()
*
- * Purpose: Decrement the flush_dep_nunser_children fields of all the
+ * Purpose: Increment the flush_dep_nunser_children fields of all the
* target entry's flush dependency parents in response to
* the target entry becoming unserialized.
*
@@ -8862,6 +8885,12 @@ H5C_remove_entry(void *_entry)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove protected entry from cache")
if(entry->is_pinned)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove pinned entry from cache")
+ /* NOTE: If these two errors are getting tripped because the entry is
+ * in a flush dependency with a freedspace entry, move the checks
+ * after the "before evict" message is sent, and add the
+ * "child being evicted" message to the "before evict" notify
+ * section below. QAK - 2017/08/03
+ */
if(entry->flush_dep_nparents > 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry with flush dependency parents from cache")
if(entry->flush_dep_nchildren > 0)
diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c
index 4a08d9b..08c70d9 100644
--- a/src/H5Cdbg.c
+++ b/src/H5Cdbg.c
@@ -716,6 +716,7 @@ H5C_stats(H5C_t * cache_ptr,
(long)(cache_ptr->LRU_list_size),
(unsigned long)(cache_ptr->LRU_list_len));
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
HDfprintf(stdout,
"%s current clean LRU size / length = %ld / %lu\n",
cache_ptr->prefix,
@@ -727,6 +728,7 @@ H5C_stats(H5C_t * cache_ptr,
cache_ptr->prefix,
(long)(cache_ptr->dLRU_list_size),
(unsigned long)(cache_ptr->dLRU_list_len));
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
HDfprintf(stdout,
"%s Total hits / misses / hit_rate = %ld / %ld / %f\n",
diff --git a/src/H5Cimage.c b/src/H5Cimage.c
index debd30c..53d1712 100644
--- a/src/H5Cimage.c
+++ b/src/H5Cimage.c
@@ -649,8 +649,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
/* Initialize fields supporting replacement policies: */
ds_entry_ptr->next = NULL;
ds_entry_ptr->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
ds_entry_ptr->aux_next = NULL;
ds_entry_ptr->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
pf_entry_ptr->coll_next = NULL;
pf_entry_ptr->coll_prev = NULL;
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index fdb14a5..d431887 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -4746,6 +4746,7 @@ struct H5C_t {
H5C_cache_entry_t * LRU_head_ptr;
H5C_cache_entry_t * LRU_tail_ptr;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
/* Fields for clean LRU list of entries */
uint32_t cLRU_list_len;
size_t cLRU_list_size;
@@ -4757,6 +4758,7 @@ struct H5C_t {
size_t dLRU_list_size;
H5C_cache_entry_t * dLRU_head_ptr;
H5C_cache_entry_t * dLRU_tail_ptr;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
/* Fields for collective metadata reads */
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 5335f80..50732ca 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -1648,8 +1648,10 @@ typedef struct H5C_cache_entry_t {
/* fields supporting replacement policies: */
struct H5C_cache_entry_t *next;
struct H5C_cache_entry_t *prev;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
struct H5C_cache_entry_t *aux_next;
struct H5C_cache_entry_t *aux_prev;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
struct H5C_cache_entry_t *coll_next;
struct H5C_cache_entry_t *coll_prev;
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index b7b8b03..af6599a 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -299,9 +299,6 @@ static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info,
static herr_t H5D__chunk_cache_prune(const H5D_t *dset, hid_t dxpl_id,
const H5D_dxpl_cache_t *dxpl_cache, size_t size);
static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk);
-static herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info,
- const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert,
- hsize_t scaled[]);
#ifdef H5_HAVE_PARALLEL
static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id,
H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf);
@@ -6558,7 +6555,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-static herr_t
+herr_t
H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk,
H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[])
{
diff --git a/src/H5Defl.c b/src/H5Defl.c
index 5536ba3..ebe7689 100644
--- a/src/H5Defl.c
+++ b/src/H5Defl.c
@@ -288,7 +288,7 @@ H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size
HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed")
if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0)
HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name")
- if((fd = HDopen(full_name, O_RDONLY, 0)) < 0)
+ if((fd = HDopen(full_name, O_RDONLY)) < 0)
HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file")
if(HDlseek(fd, (HDoff_t)(efl->slot[u].offset + (HDoff_t)skip), SEEK_SET) < 0)
HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file")
@@ -380,7 +380,7 @@ H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t siz
HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed")
if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0)
HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name")
- if((fd = HDopen(full_name, O_CREAT | O_RDWR, 0666)) < 0) {
+ if((fd = HDopen(full_name, O_CREAT | O_RDWR, H5_POSIX_CREATE_MODE_RW)) < 0) {
if(HDaccess(full_name, F_OK) < 0)
HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist")
else
diff --git a/src/H5Dint.c b/src/H5Dint.c
index 3b938e2..bdedd1e 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -1213,10 +1213,6 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
/* Don't allow compact datasets to allocate space later */
if(layout->type == H5D_COMPACT && fill->alloc_time != H5D_ALLOC_TIME_EARLY)
HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "compact dataset must have early space allocation")
-
- /* If MPI VFD is used, no filter support yet. */
- if(H5F_HAS_FEATURE(file, H5FD_FEAT_HAS_MPI) && pline->nused > 0)
- HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, NULL, "Parallel I/O does not support filters yet")
} /* end if */
/* Set the latest version of the layout, pline & fill messages, if requested */
diff --git a/src/H5Dio.c b/src/H5Dio.c
index 1766422..104a632 100644
--- a/src/H5Dio.c
+++ b/src/H5Dio.c
@@ -714,11 +714,6 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
if(H5T_get_class(type_info.mem_type, TRUE) == H5T_REFERENCE &&
H5T_get_ref_type(type_info.mem_type) == H5R_DATASET_REGION)
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing region reference datatypes yet")
-
- /* Can't write to chunked datasets with filters, in parallel */
- if(dataset->shared->layout.type == H5D_CHUNKED &&
- dataset->shared->dcpl_cache.pline.nused > 0)
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot write to chunked storage with filters in parallel")
} /* end if */
else {
/* Collective access is not permissible without a MPI based VFD */
@@ -1195,7 +1190,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id,
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator")
/* Check if we can set direct MPI-IO read/write functions */
- if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, fm, dx_plist)) < 0)
+ if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, dx_plist)) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for direct IO dataspace ")
/* Check if we can use the optimized parallel I/O routines */
@@ -1207,6 +1202,67 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id,
io_info->io_ops.single_write = H5D__mpio_select_write;
} /* end if */
else {
+ /* Check if there are any filters in the pipeline. If there are,
+ * we cannot break to independent I/O if this is a write operation;
+ * otherwise there will be metadata inconsistencies in the file.
+ */
+ if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ H5D_mpio_no_collective_cause_t cause;
+ uint32_t local_no_collective_cause;
+ uint32_t global_no_collective_cause;
+ hbool_t local_error_message_previously_written = FALSE;
+ hbool_t global_error_message_previously_written = FALSE;
+ size_t index;
+ char local_no_collective_cause_string[256] = "";
+ char global_no_collective_cause_string[256] = "";
+ const char *cause_strings[] = { "independent I/O was requested",
+ "datatype conversions were required",
+ "data transforms needed to be applied",
+ "optimized MPI types flag wasn't set",
+ "one of the dataspaces was neither simple nor scalar",
+ "dataset was not contiguous or chunked" };
+
+ if (H5P_get(dx_plist, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, &local_no_collective_cause) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get local no collective cause value")
+ if (H5P_get(dx_plist, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, &global_no_collective_cause) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get global no collective cause value")
+
+ /* Append each of the "reason for breaking collective I/O" error messages to the
+ * local and global no collective cause strings */
+ for (cause = 1, index = 0; cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE; cause <<= 1, index++) {
+ size_t cause_strlen = strlen(cause_strings[index]);
+
+ if (cause & local_no_collective_cause) {
+ /* Check if there were any previous error messages included. If so, prepend a semicolon
+ * to separate the messages.
+ */
+ if (local_error_message_previously_written) strncat(local_no_collective_cause_string, "; ", 2);
+
+ strncat(local_no_collective_cause_string, cause_strings[index], cause_strlen);
+
+ local_error_message_previously_written = TRUE;
+ } /* end if */
+
+ if (cause & global_no_collective_cause) {
+ /* Check if there were any previous error messages included. If so, prepend a semicolon
+ * to separate the messages.
+ */
+ if (global_error_message_previously_written) strncat(global_no_collective_cause_string, "; ", 2);
+
+ strncat(global_no_collective_cause_string, cause_strings[index], cause_strlen);
+
+ global_error_message_previously_written = TRUE;
+ } /* end if */
+ } /* end for */
+
+ HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, "Can't perform independent write with filters in pipeline.\n"
+ " The following caused a break from collective I/O:\n"
+ " Local causes: %s\n"
+ " Global causes: %s",
+ local_no_collective_cause_string,
+ global_no_collective_cause_string);
+ } /* end if */
+
/* If we won't be doing collective I/O, but the user asked for
* collective I/O, change the request to use independent I/O, but
* mark it so that we remember to revert the change.
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 0389c72..79572c0 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -84,7 +84,6 @@
#define H5D_CHUNK_SELECT_IRREG 2
#define H5D_CHUNK_SELECT_NONE 0
-
/******************/
/* Local Typedefs */
/******************/
@@ -94,6 +93,113 @@ typedef struct H5D_chunk_addr_info_t {
H5D_chunk_info_t chunk_info;
} H5D_chunk_addr_info_t;
+/*
+ * Information about a single chunk when performing collective filtered I/O. All
+ * of the fields of one of these structs are initialized at the start of collective
+ * filtered I/O in the function H5D__construct_filtered_io_info_list().
+ *
+ * This struct's fields are as follows:
+ *
+ * index - The "Index" of the chunk in the dataset. The index of a chunk is used during
+ * the collective re-insertion of chunks into the chunk index after the collective
+ * I/O has been performed.
+ *
+ * scaled - The scaled coordinates of the chunk in the dataset's file dataspace. The
+ * coordinates are used in both the collective re-allocation of space in the file
+ * and the collective re-insertion of chunks into the chunk index after the collective
+ * I/O has been performed.
+ *
+ * full_overwrite - A flag which determines whether or not a chunk needs to be read from the
+ * file when being updated. If a chunk is being fully overwritten (the entire
+ * extent is selected in its file dataspace), then it is not necessary to
+ * read the chunk from the file. However, if the chunk is not being fully
+ * overwritten, it has to be read from the file in order to update the chunk
+ * without trashing the parts of the chunk that are not selected.
+ *
+ * num_writers - The total number of processors writing to this chunk. This field is used
+ * when the new owner of a chunk is receiving messages, which contain selections in
+ * the chunk and data to update the chunk with, from other processors which have this
+ * chunk selected in the I/O operation. The new owner must know how many processors it
+ * should expect messages from so that it can post an equal number of receive calls.
+ *
+ * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of
+ * I/O to the chunk from each processor which has the chunk selected and is used to
+ * determine the value for the previous full_overwrite flag.
+ *
+ * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be
+ * written to the file or the chunk data which has been read from the file.
+ *
+ * chunk_states - In the case of dataset writes only, this struct is used to track a chunk's size and
+ * address in the file before and after the filtering operation has occurred.
+ *
+ * Its fields are as follows:
+ *
+ * chunk_current - The address in the file and size of this chunk before the filtering
+ * operation. When reading a chunk from the file, this field is used to
+ * read the correct amount of bytes. It is also used when redistributing
+ * shared chunks among processors and as a parameter to the chunk file
+ * space reallocation function.
+ *
+ * new_chunk - The address in the file and size of this chunk after the filtering
+ * operation. This field is relevant when collectively re-allocating space
+ * in the file for all of the chunks written to in the I/O operation, as
+ * their sizes may have changed after their data has been filtered.
+ *
+ * owners - In the case of dataset writes only, this struct is used to manage which single processor
+ * will ultimately write data out to the chunk. It allows the other processors to act according
+ * to the decision and send their selection in the chunk, as well as the data they wish
+ * to update the chunk with, to the processor which is writing to the chunk.
+ *
+ * Its fields are as follows:
+ *
+ * original_owner - The processor which originally had this chunk selected at the beginning of
+ * the collective filtered I/O operation. This field is currently used when
+ * redistributing shared chunks among processors.
+ *
+ * new_owner - The processor which has been selected to perform the write to this chunk.
+ *
+ * async_info - In the case of dataset writes only, this struct is used by the owning processor of the
+ * chunk in order to manage the MPI send and receive calls made between it and all of
+ * the other processors which have this chunk selected in the I/O operation.
+ *
+ * Its fields are as follows:
+ *
+ * receive_requests_array - An array containing one MPI_Request for each of the
+ * asynchronous MPI receive calls the owning processor of this
+ * chunk makes to another processor in order to receive that
+ * processor's chunk modification data and selection in the chunk.
+ *
+ * receive_buffer_array - An array of buffers into which the owning processor of this chunk
+ * will store chunk modification data and the selection in the chunk
+ * received from another processor.
+ *
+ * num_receive_requests - The number of entries in the receive_request_array and
+ * receive_buffer_array fields.
+ */
+typedef struct H5D_filtered_collective_io_info_t {
+ hsize_t index;
+ hsize_t scaled[H5O_LAYOUT_NDIMS];
+ hbool_t full_overwrite;
+ size_t num_writers;
+ size_t io_size;
+ void *buf;
+
+ struct {
+ H5F_block_t chunk_current;
+ H5F_block_t new_chunk;
+ } chunk_states;
+
+ struct {
+ int original_owner;
+ int new_owner;
+ } owners;
+
+ struct {
+ MPI_Request *receive_requests_array;
+ unsigned char **receive_buffer_array;
+ int num_receive_requests;
+ } async_info;
+} H5D_filtered_collective_io_info_t;
/********************/
/* Local Prototypes */
@@ -103,9 +209,15 @@ static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info,
static herr_t H5D__multi_chunk_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
H5P_genplist_t *dx_plist);
+static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
+ H5P_genplist_t *dx_plist);
static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, int sum_chunk,
H5P_genplist_t *dx_plist);
+static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
+ H5P_genplist_t *dx_plist);
static herr_t H5D__inter_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, const H5S_t *file_space,
const H5S_t *mem_space);
@@ -124,6 +236,26 @@ static herr_t H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info,
const H5D_chunk_map_t *fm, int *min_chunkf);
static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
const H5D_chunk_map_t *fm, int *sum_chunkf);
+static herr_t H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
+ H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries);
+static herr_t H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
+ H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries);
+static herr_t H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries,
+ size_t array_entry_size, void **gathered_array, size_t *gathered_array_num_entries,
+ int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *));
+static herr_t H5D__mpio_filtered_collective_write_type(
+ H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries,
+ MPI_Datatype *new_mem_type, hbool_t *mem_type_derived,
+ MPI_Datatype *new_file_type, hbool_t *file_type_derived);
+static herr_t H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry,
+ const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm);
+static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2);
+static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1,
+ const void *filtered_collective_io_info_entry2);
+static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1,
+ const void *filtered_collective_io_info_entry2);
/*********************/
@@ -142,7 +274,7 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
* Purpose: Checks if an direct I/O transfer is possible between memory and
* the file.
*
- * Return: Sauccess: Non-negative: TRUE or FALSE
+ * Return: Success: Non-negative: TRUE or FALSE
* Failure: Negative
*
* Programmer: Quincey Koziol
@@ -152,12 +284,11 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
*/
htri_t
H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
- const H5S_t *mem_space, const H5D_type_info_t *type_info,
- const H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+ const H5S_t *mem_space, const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist)
{
int local_cause = 0; /* Local reason(s) for breaking collective mode */
int global_cause = 0; /* Global reason(s) for breaking collective mode */
- htri_t ret_value; /* Return value */
+ htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@@ -206,11 +337,6 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
* use collective IO will defer until each chunk IO is reached.
*/
- /* Don't allow collective operations if filters need to be applied */
- if(io_info->dset->shared->layout.type == H5D_CHUNKED &&
- io_info->dset->shared->dcpl_cache.pline.nused > 0)
- local_cause |= H5D_MPIO_FILTERS;
-
/* Check for independent I/O */
if(local_cause & H5D_MPIO_SET_INDEPENDENT)
global_cause = local_cause;
@@ -300,6 +426,113 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5D__mpio_array_gatherv
+ *
+ * Purpose: Given an array, specified in local_array, by each processor
+ * calling this function, gathers each array into a single
+ * array which is then either gathered to the processor
+ * specified by root, when allgather is false, or is
+ * distributed back to all processors when allgather is true.
+ *
+ * The size of each entry and number of entries in the array
+ * contributed by an individual processor should be specified
+ * in array_entry_size and local_array_num_entries,
+ * respectively.
+ *
+ * The number of processors participating in the gather
+ * operation should be specified for nprocs.
+ *
+ * The MPI communicator to use should be specified for comm.
+ *
+ * If the sort_func argument is supplied, the array is sorted
+ * before the function returns.
+ *
+ * Note: if allgather is specified as true, root is ignored.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Sunday, April 9th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries,
+ size_t array_entry_size, void **_gathered_array, size_t *_gathered_array_num_entries,
+ int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *))
+{
+ size_t gathered_array_num_entries = 0; /* The size of the newly-constructed array */
+ size_t i;
+ void *gathered_array = NULL; /* The newly-constructed array returned to the caller */
+ int *receive_counts_array = NULL; /* Array containing number of entries each process is contributing */
+ int *displacements_array = NULL; /* Array of displacements where each process places its data in the final array */
+ int mpi_code;
+ int sendcount;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(_gathered_array);
+ HDassert(_gathered_array_num_entries);
+
+ /* Determine the size of the end result array */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_array_num_entries, &gathered_array_num_entries, 1, MPI_INT, MPI_SUM, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
+
+ /* If 0 entries resulted from the collective operation, no one is writing anything */
+ if (gathered_array_num_entries > 0) {
+ if (NULL == (gathered_array = H5MM_malloc(gathered_array_num_entries * array_entry_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate gathered array")
+
+ if (NULL == (receive_counts_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive counts array")
+
+ if (NULL == (displacements_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive displacements array")
+
+ /* Inform each process of how many entries each other process is contributing to the resulting array */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&local_array_num_entries, 1, MPI_INT, receive_counts_array, 1, MPI_INT, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* Multiply each receive count by the size of the array entry, since the data is sent as bytes */
+ for (i = 0; i < (size_t) nprocs; i++)
+ H5_CHECKED_ASSIGN(receive_counts_array[i], int, (size_t) receive_counts_array[i] * array_entry_size, size_t);
+
+ /* Set receive buffer offsets for MPI_Allgatherv */
+ displacements_array[0] = 0;
+ for (i = 1; i < (size_t) nprocs; i++)
+ displacements_array[i] = displacements_array[i - 1] + receive_counts_array[i - 1];
+
+ H5_CHECKED_ASSIGN(sendcount, int, local_array_num_entries * array_entry_size, size_t);
+
+ if (allgather) {
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgatherv(local_array, sendcount, MPI_BYTE,
+ gathered_array, receive_counts_array, displacements_array, MPI_BYTE, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code)
+ } /* end if */
+ else {
+ if (MPI_SUCCESS != (mpi_code = MPI_Gatherv(local_array, sendcount, MPI_BYTE,
+ gathered_array, receive_counts_array, displacements_array, MPI_BYTE, root, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code)
+ } /* end else */
+
+ if (sort_func) HDqsort(gathered_array, gathered_array_num_entries, array_entry_size, sort_func);
+ } /* end if */
+
+ *_gathered_array = gathered_array;
+ *_gathered_array_num_entries = gathered_array_num_entries;
+
+done:
+ if (receive_counts_array)
+ H5MM_free(receive_counts_array);
+ if (displacements_array)
+ H5MM_free(displacements_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__mpio_array_gatherv() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__ioinfo_xfer_mode
*
* Purpose: Switch to between collective & independent MPI I/O
@@ -398,7 +631,7 @@ H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm,
FUNC_ENTER_STATIC
/* Get the number of chunks to perform I/O on */
- num_chunkf = H5SL_count(fm->sel_chunks);
+ H5_CHECKED_ASSIGN(num_chunkf, int, H5SL_count(fm->sel_chunks), size_t)
/* Determine the minimum # of chunks for all processes */
if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&num_chunkf, min_chunkf, 1, MPI_INT, MPI_MIN, io_info->comm)))
@@ -480,7 +713,7 @@ H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish shared collective MPI-IO")
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
/* Set the actual I/O mode property. internal_collective_io will not break to
@@ -527,7 +760,7 @@ H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't finish shared collective MPI-IO")
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
/* Set the actual I/O mode property. internal_collective_io will not break to
@@ -599,12 +832,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
HDassert(fm);
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
- /* Check the optional property list on what to do with collective chunk IO. */
+ /* Check the optional property list for the collective chunk IO optimization option */
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME, &chunk_opt_mode) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option")
+
if(H5FD_MPIO_CHUNK_ONE_IO == chunk_opt_mode)
io_option = H5D_ONE_LINK_CHUNK_IO; /*no opt*/
/* direct request to multi-chunk-io */
@@ -620,13 +854,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
if((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
- /* Get the chunk optimization option */
+ /* Get the chunk optimization option threshold */
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME, &one_link_chunk_io_threshold) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option")
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option threshold value")
/* step 1: choose an IO option */
/* If the average number of chunk per process is greater than a threshold, we will do one link chunked IO. */
- if((unsigned)sum_chunk / mpi_size >= one_link_chunk_io_threshold)
+ if((unsigned)sum_chunk / (unsigned)mpi_size >= one_link_chunk_io_threshold)
io_option = H5D_ONE_LINK_CHUNK_IO_MORE_OPT;
#ifdef H5_HAVE_INSTRUMENTED_LIBRARY
else
@@ -681,19 +915,46 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
#endif
/* step 2: Go ahead to do IO.*/
- if(H5D_ONE_LINK_CHUNK_IO == io_option || H5D_ONE_LINK_CHUNK_IO_MORE_OPT == io_option) {
- if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO")
- } /* end if */
- /* direct request to multi-chunk-io */
- else if(H5D_MULTI_CHUNK_IO == io_option) {
- if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
- } /* end if */
- else { /* multiple chunk IO via threshold */
- if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
- } /* end else */
+ switch (io_option) {
+ case H5D_ONE_LINK_CHUNK_IO:
+ case H5D_ONE_LINK_CHUNK_IO_MORE_OPT:
+ /* Check if there are any filters in the pipeline */
+ if(io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ /* For now, Multi-chunk IO must be forced for parallel filtered read,
+ * so that data can be unfiltered as it is received. There is significant
+ * complexity in unfiltering the data when it is read all at once into a
+ * single buffer.
+ */
+ if (io_info->op_type == H5D_IO_OP_READ) {
+ if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO")
+ } /* end if */
+ else {
+ if(H5D__link_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish filtered linked chunk MPI-IO")
+ } /* end else */
+ } /* end if */
+ else {
+ /* Perform unfiltered link chunk collective IO */
+ if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO")
+ } /* end else */
+ break;
+
+ case H5D_MULTI_CHUNK_IO: /* direct request to do multi-chunk IO */
+ default: /* multiple chunk IO via threshold */
+ /* Check if there are any filters in the pipeline */
+ if(io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO")
+ } /* end if */
+ else {
+ /* Perform unfiltered multi chunk collective IO */
+ if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
+ } /* end else */
+ break;
+ } /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -1093,6 +1354,232 @@ if(H5DEBUG(D))
/*-------------------------------------------------------------------------
+ * Function: H5D__link_chunk_filtered_collective_io
+ *
+ * Purpose: Routine for one collective IO with one MPI derived datatype
+ * to link with all filtered chunks
+ *
+ * 1. Construct a list of selected chunks in the collective IO
+ * operation
+ * A. If any chunk is being written to by more than 1
+ * process, the process writing to the chunk which
+ * currently has the least amount of chunks assigned
+ * to it becomes the new owner (in the case of ties,
+ * the lowest MPI rank becomes the new owner)
+ * 2. If the operation is a write operation
+ * A. Loop through each chunk in the operation
+ * I. If this is not a full overwrite of the chunk
+ * a) Read the chunk from file and pass the chunk
+ * through the filter pipeline in reverse order
+ * (Unfilter the chunk)
+ * II. Update the chunk data with the modifications from
+ * the owning process
+ * III. Receive any modification data from other
+ * processes and update the chunk data with these
+ * modifications
+ * IV. Filter the chunk
+ * B. Contribute the modified chunks to an array gathered
+ * by all processes which contains the new sizes of
+ * every chunk modified in the collective IO operation
+ * C. All processes collectively re-allocate each chunk
+ * from the gathered array with their new sizes after
+ * the filter operation
+ * D. If this process has any chunks selected in the IO
+ * operation, create an MPI derived type for memory and
+ * file to write out the process' selected chunks to the
+ * file
+ * E. Perform the collective write
+ * F. All processes collectively re-insert each modified
+ * chunk from the gathered array into the chunk index
+ *
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Friday, Nov. 4th, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+{
+ H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */
+ H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */
+ H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_LINK_CHUNK; /* The actual chunk IO optimization mode */
+ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */
+ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */
+ MPI_Datatype mem_type = MPI_BYTE;
+ MPI_Datatype file_type = MPI_BYTE;
+ hbool_t mem_type_is_derived = FALSE;
+ hbool_t file_type_is_derived = FALSE;
+ size_t chunk_list_num_entries;
+ size_t collective_chunk_list_num_entries;
+ size_t *num_chunks_selected_array = NULL; /* Array of number of chunks selected on each process */
+ size_t i; /* Local index variable */
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(dx_plist);
+
+ /* Obtain the current rank of the process and the number of processes */
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ /* Set the actual-chunk-opt-mode property. */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property")
+
+ /* Set the actual-io-mode property.
+ * Link chunk filtered I/O does not break to independent, so can set right away
+ */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual io mode property")
+
+ /* Build a list of selected chunks in the collective io operation */
+ if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list")
+
+ if (io_info->op_type == H5D_IO_OP_WRITE) { /* Filtered collective write */
+ H5D_chk_idx_info_t index_info;
+ H5D_chunk_ud_t udata;
+ hsize_t mpi_buf_count;
+
+ /* Construct chunked index info */
+ index_info.f = io_info->dset->oloc.file;
+ index_info.dxpl_id = io_info->md_dxpl_id;
+ index_info.pline = &(io_info->dset->shared->dcpl_cache.pline);
+ index_info.layout = &(io_info->dset->shared->layout.u.chunk);
+ index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk);
+
+ /* Set up chunk information for insertion to chunk index */
+ udata.common.layout = index_info.layout;
+ udata.common.storage = index_info.storage;
+ udata.filter_mask = 0;
+
+ /* Iterate through all the chunks in the collective write operation,
+ * updating each chunk with the data modifications from other processes,
+ * then re-filtering the chunk.
+ */
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (mpi_rank == chunk_list[i].owners.new_owner)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry")
+
+ /* Gather the new chunk sizes to all processes for a collective reallocation
+ * of the chunks in the file.
+ */
+ if (H5D__mpio_array_gatherv(chunk_list, chunk_list_num_entries, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size,
+ true, 0, io_info->comm, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes")
+
+ /* Collectively re-allocate the modified chunks (from each process) in the file */
+ for (i = 0; i < collective_chunk_list_num_entries; i++) {
+ hbool_t insert;
+
+ if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[i].chunk_states.chunk_current,
+ &collective_chunk_list[i].chunk_states.new_chunk, &insert, collective_chunk_list[i].scaled) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+ } /* end for */
+
+ if (NULL == (num_chunks_selected_array = (size_t *) H5MM_malloc((size_t) mpi_size * sizeof(size_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, MPI_UNSIGNED_LONG_LONG, num_chunks_selected_array,
+ 1, MPI_UNSIGNED_LONG_LONG, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* If this process has any chunks selected, create a MPI type for collectively
+ * writing out the chunks to file. Otherwise, the process contributes to the
+ * collective write with a none type.
+ */
+ if (chunk_list_num_entries) {
+ size_t offset;
+
+ /* During the collective re-allocation of chunks in the file, the record for each
+ * chunk is only updated in the collective array, not in the local copy of chunks on each
+ * process. However, each process needs the updated chunk records so that they can create
+ * a MPI type for the collective write that will write to the chunk's possible new locations
+ * in the file instead of the old ones. This ugly hack seems to be the best solution to
+ * copy the information back to the local array and avoid having to modify the collective
+ * write type function in an ugly way so that it will accept the collective array instead
+ * of the local array. This works correctly because the array gather function guarantees
+ * that the chunk data in the collective array is ordered in blocks by rank.
+ */
+ for (i = 0, offset = 0; i < (size_t) mpi_rank; i++)
+ offset += num_chunks_selected_array[i];
+
+ HDmemcpy(chunk_list, &collective_chunk_list[offset], num_chunks_selected_array[mpi_rank] * sizeof(H5D_filtered_collective_io_info_t));
+
+ /* Create single MPI type encompassing each selection in the dataspace */
+ if (H5D__mpio_filtered_collective_write_type(chunk_list, chunk_list_num_entries,
+ &mem_type, &mem_type_is_derived, &file_type, &file_type_is_derived) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI link chunk I/O type")
+
+ /* Override the write buffer to point to the address of the first
+ * chunk data buffer
+ */
+ io_info->u.wbuf = chunk_list[0].buf;
+ } /* end if */
+
+ /* We have a single, complicated MPI datatype for both memory & file */
+ mpi_buf_count = (mem_type_is_derived && file_type_is_derived) ? (hsize_t) 1 : (hsize_t) 0;
+
+ /* Set up the base storage address for this operation */
+ ctg_store.contig.dset_addr = 0; /* Write address must be set to address 0 */
+ io_info->store = &ctg_store;
+
+ /* Perform I/O */
+ if (H5D__final_collective_io(io_info, type_info, mpi_buf_count, &file_type, &mem_type) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO")
+
+ /* Participate in the collective re-insertion of all chunks modified
+ * in this iteration into the chunk index
+ */
+ for (i = 0; i < collective_chunk_list_num_entries; i++) {
+ udata.chunk_block = collective_chunk_list[i].chunk_states.new_chunk;
+ udata.common.scaled = collective_chunk_list[i].scaled;
+ udata.chunk_idx = collective_chunk_list[i].index;
+
+ if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index")
+ } /* end for */
+ } /* end if */
+
+done:
+ /* Free resources used by a process which had some selection */
+ if (chunk_list) {
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (chunk_list[i].buf)
+ H5MM_free(chunk_list[i].buf);
+
+ H5MM_free(chunk_list);
+ } /* end if */
+
+ if (num_chunks_selected_array)
+ H5MM_free(num_chunks_selected_array);
+ if (collective_chunk_list)
+ H5MM_free(collective_chunk_list);
+
+ /* Free the MPI buf and file types, if they were derived */
+ if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type)))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+ if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type)))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__link_chunk_filtered_collective_io() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__multi_chunk_collective_io
*
* Purpose: To do IO per chunk according to IO mode(collective/independent/none)
@@ -1225,7 +1712,7 @@ if(H5DEBUG(D))
* to ease switching between to mixed I/O without checking the current
* value of the property. You can see the definition in H5Ppublic.h
*/
- actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE;
+ actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE);
} /* end if */
else {
@@ -1265,7 +1752,7 @@ if(H5DEBUG(D))
mspace = chunk_info->mspace;
/* Update the local variable tracking the dxpl's actual io mode. */
- actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT;
+ actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT);
} /* end if */
else {
fspace = mspace = NULL;
@@ -1306,6 +1793,314 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5D__multi_chunk_filtered_collective_io
+ *
+ * Purpose: To do filtered collective IO iteratively to save on memory.
+ * While link_chunk_filtered_collective_io will construct and
+ * work on a list of all of the chunks selected in the IO
+ * operation at once, this function works iteratively on a set
+ * of chunks at a time; at most one chunk per rank per
+ * iteration.
+ *
+ * 1. Construct a list of selected chunks in the collective IO
+ * operation
+ * A. If any chunk is being written to by more than 1
+ * process, the process writing to the chunk which
+ * currently has the least amount of chunks assigned
+ * to it becomes the new owner (in the case of ties,
+ * the lowest MPI rank becomes the new owner)
+ * 2. If the operation is a read operation
+ * A. Loop through each chunk in the operation
+ * I. Read the chunk from the file
+ * II. Unfilter the chunk
+ * III. Scatter the read chunk data to the user's buffer
+ * 3. If the operation is a write operation
+ * A. Loop through each chunk in the operation
+ * I. If this is not a full overwrite of the chunk
+ * a) Read the chunk from file and pass the chunk
+ * through the filter pipeline in reverse order
+ * (Unfilter the chunk)
+ * II. Update the chunk data with the modifications from
+ * the owning process
+ * III. Receive any modification data from other
+ * processes and update the chunk data with these
+ * modifications
+ * IV. Filter the chunk
+ * V. Contribute the chunk to an array gathered by
+ * all processes which contains every chunk
+ * modified in this iteration (up to one chunk
+ * per process, some processes may not have a
+ * selection/may have less chunks to work on than
+ * other processes)
+ * VI. All processes collectively re-allocate each
+ * chunk from the gathered array with their new
+ * sizes after the filter operation
+ * VII. Proceed with the collective write operation
+ * for the chunks modified on this iteration
+ * VIII. All processes collectively re-insert each
+ * chunk from the gathered array into the chunk
+ * index
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Friday, Dec. 2nd, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+{
+ H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */
+ H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */
+ H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_MULTI_CHUNK; /* The actual chunk IO optimization mode */
+ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */
+ H5D_storage_t store; /* union of EFL and chunk pointer in file space */
+ H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */
+ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */
+ MPI_Datatype *file_type_array = NULL;
+ MPI_Datatype *mem_type_array = NULL;
+ hbool_t *file_type_is_derived_array = NULL;
+ hbool_t *mem_type_is_derived_array = NULL;
+ hbool_t *has_chunk_selected_array = NULL; /* Array of whether or not each process is contributing a chunk to each iteration */
+ size_t chunk_list_num_entries;
+ size_t collective_chunk_list_num_entries;
+ size_t i, j; /* Local index variable */
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(dx_plist);
+
+ /* Obtain the current rank of the process and the number of processes */
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ /* Set the actual chunk opt mode property */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property")
+
+ /* Set the actual_io_mode property.
+ * Multi chunk I/O does not break to independent, so can set right away
+ */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk io mode property")
+
+ /* Build a list of selected chunks in the collective IO operation */
+ if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list")
+
+ /* Set up contiguous I/O info object */
+ HDmemcpy(&ctg_io_info, io_info, sizeof(ctg_io_info));
+ ctg_io_info.store = &ctg_store;
+ ctg_io_info.layout_ops = *H5D_LOPS_CONTIG;
+
+ /* Initialize temporary contiguous storage info */
+ ctg_store.contig.dset_size = (hsize_t) io_info->dset->shared->layout.u.chunk.size;
+ ctg_store.contig.dset_addr = 0;
+
+ /* Set dataset storage for I/O info */
+ io_info->store = &store;
+
+ if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't process chunk entry")
+ } /* end if */
+ else { /* Filtered collective write */
+ H5D_chk_idx_info_t index_info;
+ H5D_chunk_ud_t udata;
+ size_t max_num_chunks;
+ hsize_t mpi_buf_count;
+
+ /* Construct chunked index info */
+ index_info.f = io_info->dset->oloc.file;
+ index_info.dxpl_id = io_info->md_dxpl_id;
+ index_info.pline = &(io_info->dset->shared->dcpl_cache.pline);
+ index_info.layout = &(io_info->dset->shared->layout.u.chunk);
+ index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk);
+
+ /* Set up chunk information for insertion to chunk index */
+ udata.common.layout = index_info.layout;
+ udata.common.storage = index_info.storage;
+ udata.filter_mask = 0;
+
+ /* Retrieve the maximum number of chunks being written among all processes */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks,
+ 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
+
+ /* If no one is writing anything at all, end the operation */
+ if (!(max_num_chunks > 0)) HGOTO_DONE(SUCCEED);
+
+ /* Allocate arrays for storing MPI file and mem types and whether or not the
+ * types were derived.
+ */
+ if (NULL == (file_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type array")
+
+ if (NULL == (file_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type is derived array")
+
+ if (NULL == (mem_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type array")
+
+ if (NULL == (mem_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type is derived array")
+
+ /* Iterate over the max number of chunks among all processes, as this process could
+ * have no chunks left to work on, but it still needs to participate in the collective
+ * re-allocation and re-insertion of chunks modified by other processes.
+ */
+ for (i = 0; i < max_num_chunks; i++) {
+ /* Check if this process has a chunk to work on for this iteration */
+ hbool_t have_chunk_to_process = (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].owners.new_owner);
+
+ if (have_chunk_to_process)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry")
+
+ /* Gather the new chunk sizes to all processes for a collective re-allocation
+ * of the chunks in the file
+ */
+ if (H5D__mpio_array_gatherv(&chunk_list[i], have_chunk_to_process ? 1 : 0, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size,
+ true, 0, io_info->comm, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes")
+
+ /* Participate in the collective re-allocation of all chunks modified
+ * in this iteration.
+ */
+ for (j = 0; j < collective_chunk_list_num_entries; j++) {
+ hbool_t insert = FALSE;
+
+ if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[j].chunk_states.chunk_current,
+ &collective_chunk_list[j].chunk_states.new_chunk, &insert, chunk_list[j].scaled) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+ } /* end for */
+
+ if (NULL == (has_chunk_selected_array = (hbool_t *) H5MM_malloc((size_t) mpi_size * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&have_chunk_to_process, 1, MPI_C_BOOL, has_chunk_selected_array,
+ 1, MPI_C_BOOL, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* If this process has a chunk to work on, create a MPI type for the
+ * memory and file for writing out the chunk
+ */
+ if (have_chunk_to_process) {
+ size_t offset;
+ int mpi_type_count;
+
+ for (j = 0, offset = 0; j < (size_t) mpi_rank; j++)
+ offset += has_chunk_selected_array[j];
+
+ /* Collect the new chunk info back to the local copy, since only the record in the
+ * collective array gets updated by the chunk re-allocation */
+ HDmemcpy(&chunk_list[i].chunk_states.new_chunk, &collective_chunk_list[offset].chunk_states.new_chunk, sizeof(chunk_list[i].chunk_states.new_chunk));
+
+ H5_CHECKED_ASSIGN(mpi_type_count, int, chunk_list[i].chunk_states.new_chunk.length, hsize_t);
+
+ /* Create MPI memory type for writing to chunk */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &mem_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
+ if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ mem_type_is_derived_array[i] = TRUE;
+
+ /* Create MPI file type for writing to chunk */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &file_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
+ if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ file_type_is_derived_array[i] = TRUE;
+
+ mpi_buf_count = 1;
+
+ /* Set up the base storage address for this operation */
+ ctg_store.contig.dset_addr = chunk_list[i].chunk_states.new_chunk.offset;
+
+ /* Override the write buffer to point to the address of the
+ * chunk data buffer
+ */
+ ctg_io_info.u.wbuf = chunk_list[i].buf;
+ } /* end if */
+ else {
+ mem_type_array[i] = file_type_array[i] = MPI_BYTE;
+ mpi_buf_count = 0;
+ } /* end else */
+
+ /* Perform the I/O */
+ if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, &file_type_array[i], &mem_type_array[i]) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO")
+
+ /* Participate in the collective re-insertion of all chunks modified
+ * in this iteration into the chunk index
+ */
+ for (j = 0; j < collective_chunk_list_num_entries; j++) {
+ udata.chunk_block = collective_chunk_list[j].chunk_states.new_chunk;
+ udata.common.scaled = collective_chunk_list[j].scaled;
+ udata.chunk_idx = collective_chunk_list[j].index;
+
+ if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index")
+ } /* end for */
+
+ if (collective_chunk_list){
+ H5MM_free(collective_chunk_list);
+ collective_chunk_list = NULL;
+ } /* end if */
+ if (has_chunk_selected_array){
+ H5MM_free(has_chunk_selected_array);
+ has_chunk_selected_array = NULL;
+ } /* end if */
+ } /* end for */
+
+ /* Free the MPI file and memory types, if they were derived */
+ for (i = 0; i < max_num_chunks; i++) {
+ if (file_type_is_derived_array[i])
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type_array[i])))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+
+ if (mem_type_is_derived_array[i])
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type_array[i])))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+ } /* end for */
+ } /* end else */
+
+done:
+ if (chunk_list) {
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (chunk_list[i].buf)
+ H5MM_free(chunk_list[i].buf);
+
+ H5MM_free(chunk_list);
+ } /* end if */
+
+ if (collective_chunk_list)
+ H5MM_free(collective_chunk_list);
+ if (file_type_array)
+ H5MM_free(file_type_array);
+ if (mem_type_array)
+ H5MM_free(mem_type_array);
+ if (file_type_is_derived_array)
+ H5MM_free(file_type_is_derived_array);
+ if (mem_type_is_derived_array)
+ H5MM_free(mem_type_is_derived_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__multi_chunk_filtered_collective_io() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__inter_collective_io
*
* Purpose: Routine for the shared part of collective IO between multiple chunk
@@ -1472,7 +2267,7 @@ if(H5DEBUG(D))
static int
H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2)
{
- haddr_t addr1, addr2;
+ haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF;
FUNC_ENTER_STATIC_NOERR
@@ -1484,6 +2279,67 @@ H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2)
/*-------------------------------------------------------------------------
+ * Function: H5D__cmp_filtered_collective_io_info_entry
+ *
+ * Purpose: Routine to compare filtered collective chunk io info
+ * entries
+ *
+ * Description: Callback for qsort() to compare filtered collective chunk
+ * io info entries
+ *
+ * Return: -1, 0, 1
+ *
+ * Programmer: Jordan Henderson
+ * Wednesday, Nov. 30th, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2)
+{
+ haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ addr1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->chunk_states.new_chunk.offset;
+ addr2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->chunk_states.new_chunk.offset;
+
+ FUNC_LEAVE_NOAPI(H5F_addr_cmp(addr1, addr2))
+} /* end H5D__cmp_filtered_collective_io_info_entry() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__cmp_filtered_collective_io_info_entry_owner
+ *
+ * Purpose: Routine to compare filtered collective chunk io info
+ * entries's original owner fields
+ *
+ * Description: Callback for qsort() to compare filtered collective chunk
+ * io info entries's original owner fields
+ *
+ * Return: The difference between the two
+ * H5D_filtered_collective_io_info_t's original owner fields
+ *
+ * Programmer: Jordan Henderson
+ * Monday, Apr. 10th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2)
+{
+ int owner1 = -1, owner2 = -1;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ owner1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->owners.original_owner;
+ owner2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->owners.original_owner;
+
+ FUNC_LEAVE_NOAPI(owner1 - owner2)
+} /* end H5D__cmp_filtered_collective_io_info_entry_owner() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__sort_chunk
*
* Purpose: Routine to sort chunks in increasing order of chunk address
@@ -1557,7 +2413,7 @@ if(H5DEBUG(D))
HDfprintf(H5DEBUG(D), "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL\n");
#endif
/* Allocate array for chunk addresses */
- if(NULL == (total_chunk_addr_array = H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks)))
+ if(NULL == (total_chunk_addr_array = (haddr_t *)H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory chunk address array")
/* Retrieve all the chunk addresses with process 0 */
@@ -1581,7 +2437,7 @@ if(H5DEBUG(D))
/* Iterate over all chunks for this process */
while(chunk_node) {
- if(NULL == (chunk_info = H5SL_item(chunk_node)))
+ if(NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node)))
HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL,"couldn't get chunk info from skipped list")
if(many_chunk_opt == H5D_OBTAIN_ONE_CHUNK_ADDR_IND) {
@@ -1666,7 +2522,7 @@ static herr_t
H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
H5P_genplist_t *dx_plist, uint8_t assign_io_mode[], haddr_t chunk_addr[])
{
- int total_chunks;
+ size_t total_chunks;
unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk;
uint8_t* io_mode_info = NULL;
uint8_t* recv_io_mode_info = NULL;
@@ -1676,7 +2532,8 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
H5D_chunk_info_t* chunk_info;
int mpi_size, mpi_rank;
MPI_Comm comm;
- int ic, root;
+ int root;
+ size_t ic;
int mpi_code;
#ifdef H5_HAVE_INSTRUMENTED_LIBRARY
int new_value;
@@ -1697,7 +2554,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
/* Setup parameters */
- H5_CHECKED_ASSIGN(total_chunks, int, fm->layout->u.chunk.nchunks, hsize_t);
+ H5_CHECKED_ASSIGN(total_chunks, size_t, fm->layout->u.chunk.nchunks, hsize_t);
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME, &percent_nproc_per_chunk) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get percent nproc per chunk")
/* if ratio is 0, perform collective io */
@@ -1709,39 +2566,42 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
HGOTO_DONE(SUCCEED)
} /* end if */
- threshold_nproc_per_chunk = mpi_size * percent_nproc_per_chunk/100;
+
+ threshold_nproc_per_chunk = (unsigned)mpi_size * percent_nproc_per_chunk/100;
/* Allocate memory */
if(NULL == (io_mode_info = (uint8_t *)H5MM_calloc(total_chunks)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate I/O mode info buffer")
- if(NULL == (mergebuf = H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks)))
+ if(NULL == (mergebuf = (uint8_t *)H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mergebuf buffer")
tempbuf = mergebuf + total_chunks;
if(mpi_rank == root)
- if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * mpi_size)))
+ if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * (size_t)mpi_size)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate recv I/O mode info buffer")
/* Obtain the regularity and selection information for all chunks in this process. */
chunk_node = H5SL_first(fm->sel_chunks);
while(chunk_node) {
- chunk_info = H5SL_item(chunk_node);
+ chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node);
- io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */
+ io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */
chunk_node = H5SL_next(chunk_node);
} /* end while */
/* Gather all the information */
- if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, total_chunks, MPI_BYTE, recv_io_mode_info, total_chunks, MPI_BYTE, root, comm)))
+ H5_CHECK_OVERFLOW(total_chunks, size_t, int)
+ if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, (int)total_chunks, MPI_BYTE,
+ recv_io_mode_info, (int)total_chunks, MPI_BYTE, root, comm)))
HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code)
/* Calculate the mode for IO(collective, independent or none) at root process */
if(mpi_rank == root) {
- int nproc;
- int* nproc_per_chunk;
+ size_t nproc;
+ unsigned* nproc_per_chunk;
/* pre-computing: calculate number of processes and
regularity of the selection occupied in each chunk */
- if(NULL == (nproc_per_chunk = (int*)H5MM_calloc(total_chunks * sizeof(int))))
+ if(NULL == (nproc_per_chunk = (unsigned*)H5MM_calloc(total_chunks * sizeof(unsigned))))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate nproc_per_chunk buffer")
/* calculating the chunk address */
@@ -1751,7 +2611,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
} /* end if */
/* checking for number of process per chunk and regularity of the selection*/
- for(nproc = 0; nproc < mpi_size; nproc++) {
+ for(nproc = 0; nproc < (size_t)mpi_size; nproc++) {
uint8_t *tmp_recv_io_mode_info = recv_io_mode_info + (nproc * total_chunks);
/* Calculate the number of process per chunk and adding irregular selection option */
@@ -1835,5 +2695,712 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__obtain_mpio_mode() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__construct_filtered_io_info_list
+ *
+ * Purpose: Constructs a list of entries which contain the necessary
+ * information for inter-process communication when performing
+ * collective io on filtered chunks. This list is used by
+ * each process when performing I/O on locally selected chunks
+ * and also in operations that must be collectively done
+ * on every chunk, such as chunk re-allocation, insertion of
+ * chunks into the chunk index, etc.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Tuesday, January 10th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries)
+{
+ H5D_filtered_collective_io_info_t *local_info_array = NULL; /* The list of initially selected chunks for this process */
+ size_t num_chunks_selected;
+ size_t i;
+ int mpi_rank;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(chunk_list);
+ HDassert(num_entries);
+ HDassert(TRUE == H5P_isa_class(io_info->raw_dxpl_id, H5P_DATASET_XFER));
+
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+
+ /* Each process builds a local list of the chunks they have selected */
+ if ((num_chunks_selected = H5SL_count(fm->sel_chunks))) {
+ H5D_chunk_info_t *chunk_info;
+ H5D_chunk_ud_t udata;
+ H5SL_node_t *chunk_node;
+ hssize_t select_npoints;
+ hssize_t chunk_npoints;
+
+ if (NULL == (local_info_array = (H5D_filtered_collective_io_info_t *) H5MM_malloc(num_chunks_selected * sizeof(H5D_filtered_collective_io_info_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate local io info array buffer")
+
+ chunk_node = H5SL_first(fm->sel_chunks);
+ for (i = 0; chunk_node; i++) {
+ chunk_info = (H5D_chunk_info_t *) H5SL_item(chunk_node);
+
+ /* Obtain this chunk's address */
+ if (H5D__chunk_lookup(io_info->dset, io_info->md_dxpl_id, chunk_info->scaled, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+
+ local_info_array[i].index = chunk_info->index;
+ local_info_array[i].chunk_states.chunk_current = local_info_array[i].chunk_states.new_chunk = udata.chunk_block;
+ local_info_array[i].num_writers = 0;
+ local_info_array[i].owners.original_owner = local_info_array[i].owners.new_owner = mpi_rank;
+ local_info_array[i].buf = NULL;
+
+ local_info_array[i].async_info.num_receive_requests = 0;
+ local_info_array[i].async_info.receive_buffer_array = NULL;
+ local_info_array[i].async_info.receive_requests_array = NULL;
+
+ HDmemcpy(local_info_array[i].scaled, chunk_info->scaled, sizeof(chunk_info->scaled));
+
+ if ((select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ local_info_array[i].io_size = (size_t) select_npoints * type_info->src_type_size;
+
+ /* Currently the full overwrite status of a chunk is only obtained on a per-process
+ * basis. This means that if the total selection in the chunk, as determined by the combination
+ * of selections of all of the processes interested in the chunk, covers the entire chunk,
+ * the performance optimization of not reading the chunk from the file is still valid, but
+ * is not applied in the current implementation. Something like an appropriately placed
+ * MPI_Allreduce or a running total of the number of chunk points selected during chunk
+ * redistribution should suffice for implementing this case - JTH.
+ */
+ if ((chunk_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ local_info_array[i].full_overwrite =
+ (local_info_array[i].io_size >= (hsize_t) chunk_npoints * type_info->dst_type_size) ? TRUE : FALSE;
+
+ chunk_node = H5SL_next(chunk_node);
+ } /* end for */
+ } /* end if */
+
+ /* Redistribute shared chunks to new owners as necessary */
+ if (io_info->op_type == H5D_IO_OP_WRITE)
+ if (H5D__chunk_redistribute_shared_chunks(io_info, type_info, fm, local_info_array, &num_chunks_selected) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks")
+
+ *chunk_list = local_info_array;
+ *num_entries = num_chunks_selected;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__construct_filtered_io_info_list() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__chunk_redistribute_shared_chunks
+ *
+ * Purpose: When performing a collective write on a Dataset with
+ * filters applied, this function is used to redistribute any
+ * chunks which are selected by more than one process, so as
+ * to preserve file integrity after the write by ensuring
+ * that any shared chunks are only modified by one process.
+ *
+ * The current implementation follows this 3-phase process:
+ *
+ * - Collect everyone's list of chunks into one large list,
+ * sort the list in increasing order of chunk offset in the
+ * file and hand the list off to rank 0
+ *
+ * - Rank 0 scans the list looking for matching runs of chunk
+ * offset in the file (corresponding to a shared chunk which
+ * has been selected by more than one rank in the I/O
+ * operation) and for each shared chunk, it redistributes
+ * the chunk to the process writing to the chunk which
+ * currently has the least amount of chunks assigned to it
+ * by modifying the "new_owner" field in each of the list
+ * entries corresponding to that chunk
+ *
+ * - After the chunks have been redistributed, rank 0 re-sorts
+ * the list in order of previous owner so that each rank
+ * will get back exactly the array that they contributed to
+ * the redistribution operation, with the "new_owner" field
+ * of each chunk they are modifying having possibly been
+ * modified. Rank 0 then scatters each segment of the list
+ * back to its corresponding rank
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Monday, May 1, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries)
+{
+ H5D_filtered_collective_io_info_t *shared_chunks_info_array = NULL; /* The list of all chunks selected in the operation by all processes */
+ H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__gather_mem */
+ unsigned char **mod_data = NULL; /* Array of chunk modification data buffers sent by a process to new chunk owners */
+ MPI_Request *send_requests = NULL; /* Array of MPI_Isend chunk modification data send requests */
+ MPI_Status *send_statuses = NULL; /* Array of MPI_Isend chunk modification send statuses */
+ hbool_t mem_iter_init = FALSE;
+ size_t shared_chunks_info_array_num_entries = 0;
+ size_t num_send_requests = 0;
+ size_t *num_assigned_chunks_array = NULL;
+ size_t i, last_assigned_idx;
+ int *send_counts = NULL;
+ int *send_displacements = NULL;
+ int scatter_recvcount_int;
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(local_chunk_array_num_entries);
+
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ if (*local_chunk_array_num_entries)
+ if (NULL == (send_requests = (MPI_Request *) H5MM_malloc(*local_chunk_array_num_entries * sizeof(MPI_Request))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests buffer")
+
+ if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator")
+
+ /* Gather every rank's list of chunks to rank 0 to allow it to perform the redistribution operation. After this
+ * call, the gathered list will initially be sorted in increasing order of chunk offset in the file.
+ */
+ if (H5D__mpio_array_gatherv(local_chunk_array, *local_chunk_array_num_entries, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &shared_chunks_info_array, &shared_chunks_info_array_num_entries, mpi_size,
+ false, 0, io_info->comm, H5D__cmp_filtered_collective_io_info_entry) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather array")
+
+ /* Rank 0 redistributes any shared chunks to new owners as necessary */
+ if (mpi_rank == 0) {
+ if (NULL == (send_counts = (int *) H5MM_calloc((size_t) mpi_size * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send counts buffer")
+
+ if (NULL == (send_displacements = (int *) H5MM_malloc((size_t) mpi_size * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send displacements buffer")
+
+ if (NULL == (num_assigned_chunks_array = (size_t *) H5MM_calloc((size_t) mpi_size * sizeof(size_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate number of assigned chunks array")
+
+ for (i = 0; i < shared_chunks_info_array_num_entries;) {
+ H5D_filtered_collective_io_info_t chunk_entry;
+ haddr_t last_seen_addr = shared_chunks_info_array[i].chunk_states.chunk_current.offset;
+ size_t set_begin_index = i;
+ size_t num_writers = 0;
+ int new_chunk_owner = shared_chunks_info_array[i].owners.original_owner;
+
+ /* Process each set of duplicate entries caused by another process writing to the same chunk */
+ do {
+ chunk_entry = shared_chunks_info_array[i];
+
+ send_counts[chunk_entry.owners.original_owner] += (int) sizeof(chunk_entry);
+
+ /* The new owner of the chunk is determined by the process
+ * writing to the chunk which currently has the least amount
+ * of chunks assigned to it
+ */
+ if (num_assigned_chunks_array[chunk_entry.owners.original_owner] < num_assigned_chunks_array[new_chunk_owner])
+ new_chunk_owner = chunk_entry.owners.original_owner;
+
+ num_writers++;
+ } while (++i < shared_chunks_info_array_num_entries && shared_chunks_info_array[i].chunk_states.chunk_current.offset == last_seen_addr);
+
+ /* Set all of the chunk entries' "new_owner" fields */
+ for (; set_begin_index < i; set_begin_index++) {
+ shared_chunks_info_array[set_begin_index].owners.new_owner = new_chunk_owner;
+ shared_chunks_info_array[set_begin_index].num_writers = num_writers;
+ } /* end for */
+
+ num_assigned_chunks_array[new_chunk_owner]++;
+ } /* end for */
+
+ /* Sort the new list in order of previous owner so that each original owner of a chunk
+ * entry gets that entry back, with the possibly newly-modified "new_owner" field
+ */
+ HDqsort(shared_chunks_info_array, shared_chunks_info_array_num_entries,
+ sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry_owner);
+
+ send_displacements[0] = 0;
+ for (i = 1; i < (size_t) mpi_size; i++)
+ send_displacements[i] = send_displacements[i - 1] + send_counts[i - 1];
+ } /* end if */
+
+ /* Scatter the segments of the list back to each process */
+ H5_CHECKED_ASSIGN(scatter_recvcount_int, int, *local_chunk_array_num_entries * sizeof(H5D_filtered_collective_io_info_t), size_t);
+ if (MPI_SUCCESS != (mpi_code = MPI_Scatterv(shared_chunks_info_array, send_counts, send_displacements,
+ MPI_BYTE, local_chunk_array, scatter_recvcount_int, MPI_BYTE, 0, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code)
+
+ if (shared_chunks_info_array) {
+ H5MM_free(shared_chunks_info_array);
+ shared_chunks_info_array = NULL;
+ } /* end if */
+
+ /* Now that the chunks have been redistributed, each process must send its modification data
+ * to the new owners of any of the chunks it previously possessed. Accordingly, each process
+ * must also issue asynchronous receives for any messages it may receive for each of the
+ * chunks it is assigned, in order to avoid potential deadlocking issues.
+ */
+ if (*local_chunk_array_num_entries)
+ if (NULL == (mod_data = (unsigned char **) H5MM_malloc(*local_chunk_array_num_entries * sizeof(unsigned char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data buffer array")
+
+ for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) {
+ H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i];
+
+ if (mpi_rank != chunk_entry->owners.new_owner) {
+ H5D_chunk_info_t *chunk_info = NULL;
+ unsigned char *mod_data_p = NULL;
+ hssize_t iter_nelmts;
+ size_t mod_data_size;
+
+ /* Look up the chunk and get its file and memory dataspaces */
+ if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list")
+
+ /* Determine size of serialized chunk file dataspace, plus the size of
+ * the data being written
+ */
+ if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to get encoded dataspace size")
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ mod_data_size += (size_t) iter_nelmts * type_info->src_type_size;
+
+ if (NULL == (mod_data[num_send_requests] = (unsigned char *) H5MM_malloc(mod_data_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk modification send buffer")
+
+ /* Serialize the chunk's file dataspace into the buffer */
+ mod_data_p = mod_data[num_send_requests];
+ if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace")
+
+ /* Intialize iterator for memory selection */
+ if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ /* Collect the modification data into the buffer */
+ if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, mod_data_p))
+ HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer")
+
+ /* Send modification data to new owner */
+ H5_CHECK_OVERFLOW(mod_data_size, size_t, int)
+ H5_CHECK_OVERFLOW(chunk_entry->index, hsize_t, int)
+ if (MPI_SUCCESS != (mpi_code = MPI_Isend(mod_data[num_send_requests], (int) mod_data_size, MPI_BYTE,
+ chunk_entry->owners.new_owner, (int) chunk_entry->index, io_info->comm, &send_requests[num_send_requests])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Isend failed", mpi_code)
+
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator")
+ mem_iter_init = FALSE;
+
+ num_send_requests++;
+ } /* end if */
+ else {
+ /* Allocate all necessary buffers for an asynchronous receive operation */
+ if (chunk_entry->num_writers > 1) {
+ MPI_Message message;
+ MPI_Status status;
+ size_t j;
+
+ chunk_entry->async_info.num_receive_requests = (int) chunk_entry->num_writers - 1;
+ if (NULL == (chunk_entry->async_info.receive_requests_array = (MPI_Request *) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(MPI_Request))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async requests array")
+
+ if (NULL == (chunk_entry->async_info.receive_buffer_array = (unsigned char **) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(unsigned char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async receive buffers")
+
+ for (j = 0; j < chunk_entry->num_writers - 1; j++) {
+ int count = 0;
+
+ /* Probe for a particular message from any process, removing that message
+ * from the receive queue in the process and allocating that much memory
+ * for the asynchronous receive
+ */
+ if (MPI_SUCCESS != (mpi_code = MPI_Mprobe(MPI_ANY_SOURCE, (int) chunk_entry->index, io_info->comm, &message, &status)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Mprobe failed", mpi_code)
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_count(&status, MPI_BYTE, &count)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mpi_code)
+
+ HDassert(count >= 0);
+ if (NULL == (chunk_entry->async_info.receive_buffer_array[j] = (unsigned char *) H5MM_malloc((size_t) count * sizeof(char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data receive buffer")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Imrecv(chunk_entry->async_info.receive_buffer_array[j], count, MPI_BYTE,
+ &message, &chunk_entry->async_info.receive_requests_array[j])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Imrecv failed", mpi_code)
+ } /* end for */
+ } /* end if */
+
+ local_chunk_array[last_assigned_idx++] = local_chunk_array[i];
+ } /* end else */
+ } /* end for */
+
+ *local_chunk_array_num_entries = last_assigned_idx;
+
+ /* Wait for all async send requests to complete before returning */
+ if (num_send_requests) {
+ if (NULL == (send_statuses = (MPI_Status *) H5MM_malloc(num_send_requests * sizeof(MPI_Status))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send statuses buffer")
+
+ H5_CHECK_OVERFLOW(num_send_requests, size_t, int);
+ if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int) num_send_requests, send_requests, send_statuses)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code)
+ } /* end if */
+
+done:
+ /* Now that all async send requests have completed, free up the send
+ * buffers used in the async operations
+ */
+ for (i = 0; i < num_send_requests; i++) {
+ if (mod_data[i])
+ H5MM_free(mod_data[i]);
+ } /* end for */
+
+ if (send_requests)
+ H5MM_free(send_requests);
+ if (send_statuses)
+ H5MM_free(send_statuses);
+ if (send_counts)
+ H5MM_free(send_counts);
+ if (send_displacements)
+ H5MM_free(send_displacements);
+ if (mod_data)
+ H5MM_free(mod_data);
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ if (mem_iter)
+ H5MM_free(mem_iter);
+ if (num_assigned_chunks_array)
+ H5MM_free(num_assigned_chunks_array);
+ if (shared_chunks_info_array)
+ H5MM_free(shared_chunks_info_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__chunk_redistribute_shared_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__mpio_filtered_collective_write_type
+ *
+ * Purpose: Constructs a MPI derived datatype for both the memory and
+ * the file for a collective write of filtered chunks. The
+ * datatype contains the offsets in the file and the locations
+ * of the filtered chunk data buffers.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Tuesday, November 22, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list,
+ size_t num_entries, MPI_Datatype *new_mem_type, hbool_t *mem_type_derived,
+ MPI_Datatype *new_file_type, hbool_t *file_type_derived)
+{
+ MPI_Aint *write_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */
+ MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */
+ int *length_array = NULL; /* Filtered Chunk lengths */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(chunk_list);
+ HDassert(new_mem_type);
+ HDassert(mem_type_derived);
+ HDassert(new_file_type);
+ HDassert(file_type_derived);
+
+ if (num_entries > 0) {
+ size_t i;
+ int mpi_code;
+ void *base_buf;
+
+ H5_CHECK_OVERFLOW(num_entries, size_t, int);
+
+ /* Allocate arrays */
+ if (NULL == (length_array = (int *) H5MM_malloc((size_t) num_entries * sizeof(int))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write length array")
+ if (NULL == (write_buf_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write buf length array")
+ if (NULL == (file_offset_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for collective write offset array")
+
+ /* Ensure the list is sorted in ascending order of offset in the file */
+ HDqsort(chunk_list, num_entries, sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry);
+
+ base_buf = chunk_list[0].buf;
+ for (i = 0; i < num_entries; i++) {
+ /* Set up the offset in the file, the length of the chunk data, and the relative
+ * displacement of the chunk data write buffer
+ */
+ file_offset_array[i] = (MPI_Aint) chunk_list[i].chunk_states.new_chunk.offset;
+ length_array[i] = (int) chunk_list[i].chunk_states.new_chunk.length;
+ write_buf_array[i] = (MPI_Aint) chunk_list[i].buf - (MPI_Aint) base_buf;
+ } /* end for */
+
+ /* Create memory MPI type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, write_buf_array, MPI_BYTE, new_mem_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
+ *mem_type_derived = TRUE;
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+
+ /* Create file MPI type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, file_offset_array, MPI_BYTE, new_file_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
+ *file_type_derived = TRUE;
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ } /* end if */
+
+done:
+ if (write_buf_array)
+ H5MM_free(write_buf_array);
+ if (file_offset_array)
+ H5MM_free(file_offset_array);
+ if (length_array)
+ H5MM_free(length_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__mpio_filtered_collective_write_type() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__filtered_collective_chunk_entry_io
+ *
+ * Purpose: Given an entry for a filtered chunk, performs the necessary
+ * steps for updating the chunk data during a collective
+ * write, or for reading the chunk from file during a
+ * collective read.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Wednesday, January 18, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry,
+ const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm)
+{
+ H5D_chunk_info_t *chunk_info = NULL;
+ H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__scatter_mem/H5D__gather_mem */
+ unsigned char *mod_data = NULL; /* Chunk modification data sent by a process to a chunk's owner */
+ unsigned filter_mask = 0;
+ hssize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */
+ hssize_t extent_npoints;
+ hsize_t true_chunk_size;
+ hbool_t mem_iter_init = FALSE;
+ size_t buf_size;
+ size_t i;
+ H5S_t *dataspace = NULL; /* Other process' dataspace for the chunk */
+ void *tmp_gath_buf = NULL; /* Temporary gather buffer for owner of the chunk to gather into from
+ application write buffer before scattering out to the chunk data buffer */
+ int mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(chunk_entry);
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+
+ /* Look up the chunk and get its file and memory dataspaces */
+ if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list")
+
+ if ((extent_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ true_chunk_size = (hsize_t) extent_npoints * type_info->src_type_size;
+
+ /* If the size of the filtered chunk is larger than the number of points in the
+ * chunk file space extent times the datatype size, allocate enough space to hold the
+ * whole filtered chunk. Otherwise, allocate a buffer equal to the size of the
+ * chunk so that the unfiltering operation doesn't have to grow the buffer.
+ */
+ buf_size = MAX(chunk_entry->chunk_states.chunk_current.length, true_chunk_size);
+
+ if (NULL == (chunk_entry->buf = H5MM_malloc(buf_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer")
+
+ /* If this is not a full chunk overwrite or this is a read operation, the chunk must be
+ * read from the file and unfiltered.
+ */
+ if (!chunk_entry->full_overwrite || io_info->op_type == H5D_IO_OP_READ) {
+ chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length;
+
+ /* Currently, these chunk reads are done independently and will likely
+ * cause issues with collective metadata reads enabled. In the future,
+ * this should be refactored to use collective chunk reads - JTH */
+ if (H5F_block_read(io_info->dset->oloc.file, H5FD_MEM_DRAW, chunk_entry->chunk_states.chunk_current.offset,
+ chunk_entry->chunk_states.new_chunk.length, H5AC_rawdata_dxpl_id, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk")
+
+ if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, &filter_mask,
+ io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb,
+ (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying")
+ } /* end if */
+ else {
+ chunk_entry->chunk_states.new_chunk.length = true_chunk_size;
+ } /* end else */
+
+ /* Initialize iterator for memory selection */
+ if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator")
+
+ if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* If this is a read operation, scatter the read chunk data to the user's buffer.
+ *
+ * If this is a write operation, update the chunk data buffer with the modifications
+ * from the current process, then apply any modifications from other processes. Finally,
+ * filter the newly-updated chunk.
+ */
+ switch (io_info->op_type) {
+ case H5D_IO_OP_READ:
+ if (H5D__scatter_mem(chunk_entry->buf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, io_info->u.rbuf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to read buffer")
+ break;
+
+ case H5D_IO_OP_WRITE:
+ if (NULL == (tmp_gath_buf = H5MM_malloc((hsize_t) iter_nelmts * type_info->src_type_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer")
+
+ /* Gather modification data from the application write buffer into a temporary buffer */
+ if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, tmp_gath_buf))
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't gather from write buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+
+ /* Initialize iterator for file selection */
+ if (H5S_select_iter_init(mem_iter, chunk_info->fspace, type_info->dst_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize file selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* Scatter the owner's modification data into the chunk data buffer according to
+ * the file space.
+ */
+ if (H5D__scatter_mem(tmp_gath_buf, chunk_info->fspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to chunk data buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Waitall(chunk_entry->async_info.num_receive_requests,
+ chunk_entry->async_info.receive_requests_array, MPI_STATUSES_IGNORE)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code)
+
+ /* For each asynchronous receive call previously posted, receive the chunk modification
+ * buffer from another rank and update the chunk data
+ */
+ for (i = 0; i < (size_t) chunk_entry->async_info.num_receive_requests; i++) {
+ const unsigned char *mod_data_p;
+
+ /* Decode the process' chunk file dataspace */
+ mod_data_p = chunk_entry->async_info.receive_buffer_array[i];
+ if (NULL == (dataspace = H5S_decode(&mod_data_p)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace")
+
+ if (H5S_select_iter_init(mem_iter, dataspace, type_info->dst_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* Update the chunk data with the received modification data */
+ if (H5D__scatter_mem(mod_data_p, dataspace, mem_iter, (size_t) iter_nelmts,
+ io_info->dxpl_cache, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+ if (dataspace) {
+ if (H5S_close(dataspace) < 0)
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace")
+ dataspace = NULL;
+ }
+ H5MM_free(chunk_entry->async_info.receive_buffer_array[i]);
+ } /* end for */
+
+ /* Filter the chunk */
+ if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, &filter_mask,
+ io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb,
+ (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed")
+
+#if H5_SIZEOF_SIZE_T > 4
+ /* Check for the chunk expanding too much to encode in a 32-bit value */
+ if (chunk_entry->chunk_states.new_chunk.length > ((size_t) 0xffffffff))
+ HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length")
+#endif
+
+ break;
+ default:
+ HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid I/O operation")
+ } /* end switch */
+
+done:
+ if (chunk_entry->async_info.receive_buffer_array)
+ H5MM_free(chunk_entry->async_info.receive_buffer_array);
+ if (chunk_entry->async_info.receive_requests_array)
+ H5MM_free(chunk_entry->async_info.receive_requests_array);
+ if (mod_data)
+ H5MM_free(mod_data);
+ if (tmp_gath_buf)
+ H5MM_free(tmp_gath_buf);
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ if (mem_iter)
+ H5MM_free(mem_iter);
+ if (dataspace)
+ if (H5S_close(dataspace) < 0)
+ HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace")
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__filtered_collective_chunk_entry_io() */
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index a6857b9..097fab7 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -617,6 +617,9 @@ H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info,
H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf,
const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
const H5D_dxpl_cache_t *dxpl_cache, void *_buf);
+H5_DLL size_t H5D__gather_mem(const void *_buf,
+ const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
+ const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/);
H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info,
const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
@@ -666,6 +669,8 @@ H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id,
const hsize_t *scaled, H5D_chunk_ud_t *udata);
H5_DLL herr_t H5D__chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes);
H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]);
+H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk,
+ H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[]);
H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id,
hsize_t old_dim[]);
H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id,
@@ -768,8 +773,7 @@ H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info,
* memory and the file */
H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info,
const H5S_t *file_space, const H5S_t *mem_space,
- const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
- H5P_genplist_t *dx_plist);
+ const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist);
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c
index 4625c7a..0ae69ee 100644
--- a/src/H5Dscatgath.c
+++ b/src/H5Dscatgath.c
@@ -47,9 +47,6 @@ static herr_t H5D__scatter_file(const H5D_io_info_t *io_info,
static size_t H5D__gather_file(const H5D_io_info_t *io_info,
const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts,
void *buf);
-static size_t H5D__gather_mem(const void *_buf,
- const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
- const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/);
static herr_t H5D__compound_opt_read(size_t nelmts, const H5S_t *mem_space,
H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache,
const H5D_type_info_t *type_info, void *user_buf/*out*/);
@@ -303,6 +300,7 @@ H5D__scatter_mem (const void *_tscat_buf, const H5S_t *space,
HDassert(space);
HDassert(iter);
HDassert(nelmts > 0);
+ HDassert(dxpl_cache);
HDassert(buf);
/* Allocate the vector I/O arrays */
@@ -364,7 +362,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-static size_t
+size_t
H5D__gather_mem(const void *_buf, const H5S_t *space,
H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache,
void *_tgath_buf/*out*/)
@@ -387,6 +385,7 @@ H5D__gather_mem(const void *_buf, const H5S_t *space,
HDassert(space);
HDassert(iter);
HDassert(nelmts > 0);
+ HDassert(dxpl_cache);
HDassert(tgath_buf);
/* Allocate the vector I/O arrays */
diff --git a/src/H5F.c b/src/H5F.c
index 78fce2a..39eca13 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -443,8 +443,6 @@ done:
hid_t
H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
{
- hbool_t ci_load = FALSE; /* whether MDC ci load requested */
- hbool_t ci_write = FALSE; /* whether MDC CI write requested */
H5F_t *new_file = NULL; /*file struct for new file */
hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */
hid_t ret_value; /*return value */
@@ -490,12 +488,6 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
if(NULL == (new_file = H5F_open(filename, flags, fcpl_id, fapl_id, dxpl_id)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file")
- /* Check to see if both SWMR and cache image are requested. Fail if so */
- if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status")
- if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
- HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image")
-
/* Get an atom for the file */
if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file")
@@ -554,8 +546,6 @@ done:
hid_t
H5Fopen(const char *filename, unsigned flags, hid_t fapl_id)
{
- hbool_t ci_load = FALSE; /* whether MDC ci load requested */
- hbool_t ci_write = FALSE; /* whether MDC CI write requested */
H5F_t *new_file = NULL; /*file struct for new file */
hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */
hid_t ret_value; /*return value */
@@ -585,12 +575,6 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id)
if(NULL == (new_file = H5F_open(filename, flags, H5P_FILE_CREATE_DEFAULT, fapl_id, dxpl_id)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to open file")
- /* Check to see if both SWMR and cache image are requested. Fail if so */
- if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status")
- if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
- HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image")
-
/* Get an atom for the file */
if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle")
@@ -1633,7 +1617,7 @@ H5Fstart_swmr_write(hid_t file_id)
if(file->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "file superblock version should be at least 3")
- HDassert(file->shared->latest_flags == H5F_LATEST_ALL_FLAGS);
+ HDassert((file->shared->latest_flags | H5F_LATEST_LAYOUT_MSG) > 0);
/* Should not be marked for SWMR writing mode already */
if(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)
@@ -1647,6 +1631,10 @@ H5Fstart_swmr_write(hid_t file_id)
if(ci_load || ci_write )
HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image")
+ /* Flush the superblock extension */
+ if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, H5AC_ind_read_dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock extension")
+
/* Flush data buffers */
if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index b980b7e..2ab04dc 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -598,7 +598,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
{
int o_flags;
H5FD_core_t *file = NULL;
- H5FD_core_fapl_t *fa = NULL;
+ const H5FD_core_fapl_t *fa = NULL;
H5P_genplist_t *plist; /* Property list pointer */
#ifdef H5_HAVE_WIN32_API
struct _BY_HANDLE_FILE_INFORMATION fileinfo;
@@ -620,7 +620,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
HDassert(H5P_DEFAULT != fapl_id);
if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
- if(NULL == (fa = (H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
+ if(NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info")
/* Build the open flags */
@@ -638,7 +638,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
((file_image_info.buffer == NULL) && (file_image_info.size == 0)));
HDmemset(&sb, 0, sizeof(sb));
if((file_image_info.buffer != NULL) && !(H5F_ACC_CREAT & flags)) {
- if(HDopen(name, o_flags, 0666) >= 0)
+ if(HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW) >= 0)
HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file already exists")
/* If backing store is requested, create and stat the file
@@ -646,7 +646,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
* technically an open.
*/
if(fa->backing_store) {
- if((fd = HDopen(name, o_flags | O_CREAT, 0666)) < 0)
+ if((fd = HDopen(name, o_flags | O_CREAT, H5_POSIX_CREATE_MODE_RW)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create file")
if(HDfstat(fd, &sb) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file")
@@ -656,7 +656,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
* store is off is when the backing_store flag is off and H5F_ACC_CREAT is
* on. */
else if(fa->backing_store || !(H5F_ACC_CREAT & flags)) {
- if((fd = HDopen(name, o_flags, 0666)) < 0)
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file")
if(HDfstat(fd, &sb) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file")
diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c
index 7a4c99c..811ea8e 100644
--- a/src/H5FDdirect.c
+++ b/src/H5FDdirect.c
@@ -487,7 +487,7 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd
o_flags |= O_DIRECT;
/* Open the file */
- if ((fd=HDopen(name, o_flags, 0666))<0)
+ if ((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW))<0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file")
if (HDfstat(fd, &sb)<0)
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 7c6bbd4..7c2297a 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -520,7 +520,7 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
HDgettimeofday(&timeval_start, NULL);
#endif /* H5_HAVE_GETTIMEOFDAY */
/* Open the file */
- if((fd = HDopen(name, o_flags, 0666)) < 0) {
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) {
int myerrno = errno;
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags);
diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c
index 16be455..98e1b1a 100644
--- a/src/H5FDmpi.c
+++ b/src/H5FDmpi.c
@@ -315,6 +315,11 @@ H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, MPI_Comm *comm_new, MPI_Inf
info_dup = info;
}
+ /* Set MPI_ERRORS_RETURN on comm_dup so that MPI failures are not fatal,
+ and return codes can be checked and handled. May 23, 2017 FTW */
+ if (MPI_SUCCESS != (mpi_code = MPI_Comm_set_errhandler(comm_dup, MPI_ERRORS_RETURN)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Errhandler_set failed", mpi_code)
+
/* copy them to the return arguments */
*comm_new = comm_dup;
*info_new = info_dup;
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index b6385fb..32aff0f 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -341,7 +341,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
o_flags |= O_EXCL;
/* Open the file */
- if((fd = HDopen(name, o_flags, 0666)) < 0) {
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) {
int myerrno = errno;
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags);
} /* end if */
diff --git a/src/H5FSint.c b/src/H5FSint.c
index 1a41172..4297291 100644
--- a/src/H5FSint.c
+++ b/src/H5FSint.c
@@ -77,6 +77,32 @@
/*******************/
+/*-------------------------------------------------------------------------
+ * Function: H5FS_init
+ *
+ * Purpose: Initialize the interface in case it is unable to initialize
+ * itself soon enough.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Saturday, March 4, 2000
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FS_init(void)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+ /* FUNC_ENTER() does all the work */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FS_init() */
+
/*-------------------------------------------------------------------------
* Function: H5FS__create_flush_depend
diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h
index c0467a6..247d75c 100644
--- a/src/H5FSprivate.h
+++ b/src/H5FSprivate.h
@@ -175,6 +175,9 @@ H5FL_SEQ_EXTERN(H5FS_section_class_t);
/* Library-private Function Prototypes */
/***************************************/
+/* Package initialization routine */
+H5_DLL herr_t H5FS_init(void);
+
/* Free space manager routines */
H5_DLL H5FS_t *H5FS_create(H5F_t *f, hid_t dxpl_id, haddr_t *fs_addr,
const H5FS_create_t *fs_create, uint16_t nclasses,
diff --git a/src/H5Fint.c b/src/H5Fint.c
index e52d539..7371c46 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -678,8 +678,10 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
if(H5P_get(plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'latest format' flag")
/* For latest format or SWMR_WRITE, activate all latest version support */
- if(latest_format || (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE))
+ if(latest_format)
f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
+ else if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
+ f->shared->latest_flags |= H5F_LATEST_LAYOUT_MSG;
if(H5P_get(plist, H5F_ACS_USE_MDC_LOGGING_NAME, &(f->shared->use_mdc_logging)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'use mdc logging' flag")
if(H5P_get(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, &(f->shared->start_mdc_log_on_access)) < 0)
@@ -1184,6 +1186,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
H5F_t *ret_value = NULL; /*actual return value */
char *lock_env_var = NULL;/*env var pointer */
hbool_t use_file_locking; /*read from env var */
+ hbool_t ci_load = FALSE; /* whether MDC ci load requested */
+ hbool_t ci_write = FALSE; /* whether MDC CI write requested */
FUNC_ENTER_NOAPI(NULL)
@@ -1317,6 +1321,12 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
set_flag = TRUE;
} /* end else */
+ /* Check to see if both SWMR and cache image are requested. Fail if so */
+ if(H5C_cache_image_status(file, &ci_load, &ci_write) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get MDC cache image status")
+ if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
+ HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, NULL, "can't have both SWMR and cache image")
+
/* Retain the name the file was opened with */
file->open_name = H5MM_xstrdup(name);
@@ -1456,11 +1466,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)
file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS;
- /* Flush the superblock */
+ /* Flush the superblock & superblock extension */
if(H5F_super_dirty(file) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty")
if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock")
+ if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, meta_dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock extension")
/* Remove the file lock for SWMR_WRITE */
if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) {
@@ -1493,6 +1505,7 @@ done:
if((NULL == ret_value) && file)
if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file")
+
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_open() */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 7a5c126..d702506 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -309,16 +309,16 @@ struct H5F_file_t {
H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */
/* File space allocation information */
- H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */
+ H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */
hsize_t fs_threshold; /* Free space section threshold */
- hbool_t fs_persist; /* Free-space persist or not */
+ hbool_t fs_persist; /* Free-space persist or not */
hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */
haddr_t tmp_addr; /* Next address to use for temp. space in the file */
- hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */
+ hbool_t point_of_no_return; /* Flag to indicate that we can't go back and delete a freespace header when it's used up */
H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */
- haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
- H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
+ haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
+ H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
hbool_t first_alloc_dealloc; /* TRUE iff free space managers */
/* are persistant and have not */
/* been used accessed for either */
@@ -333,25 +333,25 @@ struct H5F_file_t {
/* HADDR_UNDEF if no cache image. */
/* Free-space aggregation info */
- unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */
- H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
- H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */
- H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */
+ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */
+ H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
+ H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */
+ H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */
/* Paged aggregation info */
- hsize_t fs_page_size; /* File space page size */
- size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */
+ hsize_t fs_page_size; /* File space page size */
+ size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */
/* Metadata accumulator information */
- H5F_meta_accum_t accum; /* Metadata accumulator info */
+ H5F_meta_accum_t accum; /* Metadata accumulator info */
/* Metadata retry info */
- unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */
- unsigned retries_nbins; /* # of bins for each retries[] */
+ unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */
+ unsigned retries_nbins; /* # of bins for each retries[] */
uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */
/* Object flush info */
- H5F_object_flush_t object_flush; /* Information for object flush callback */
+ H5F_object_flush_t object_flush; /* Information for object flush callback */
};
/*
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 7c70a64..0c6f9cd 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -21,15 +21,15 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
+#include "H5private.h" /* Generic Functions */
#include "H5ACprivate.h" /* Metadata cache */
-#include "H5Eprivate.h" /* Error handling */
+#include "H5Eprivate.h" /* Error handling */
#include "H5Fpkg.h" /* File access */
-#include "H5FDprivate.h" /* File drivers */
-#include "H5Iprivate.h" /* IDs */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5Iprivate.h" /* IDs */
#include "H5MFprivate.h" /* File memory management */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
#include "H5SMprivate.h" /* Shared Object Header Messages */
@@ -158,7 +158,7 @@ H5F_super_ext_open(H5F_t *f, haddr_t ext_addr, H5O_loc_t *ext_ptr)
/* Open the superblock extension object header */
if(H5O_open(ext_ptr) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension")
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -224,12 +224,12 @@ done:
/*-------------------------------------------------------------------------
* Function: H5F__update_super_ext_driver_msg
*
- * Purpose: Update the superblock extension file driver info message if
- * we are using a V 2 superblock. Observe that the function
- * is a NO-OP if the file driver info message does not exist.
+ * Purpose: Update the superblock extension file driver info message if
+ * we are using a V 2 superblock. Observe that the function
+ * is a NO-OP if the file driver info message does not exist.
* This is necessary, as the function is called whenever the
- * EOA is updated, and were it to create the file driver info
- * message, it would find itself in an infinite recursion.
+ * EOA is updated, and were it to create the file driver info
+ * message, it would find itself in an infinite recursion.
*
* Return: Success: SUCCEED
* Failure: FAIL
@@ -267,7 +267,7 @@ H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id)
/* Check for driver info */
H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
- /* Nothing to do unless there is both driver info and
+ /* Nothing to do unless there is both driver info and
* the driver info superblock extension message has
* already been created.
*/
@@ -330,9 +330,13 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */
haddr_t super_addr; /* Absolute address of superblock */
haddr_t eof; /* End of file address */
- unsigned rw_flags; /* Read/write permissions for file */
- hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
+ unsigned rw_flags; /* Read/write permissions for file */
+ hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
herr_t ret_value = SUCCEED; /* Return value */
+#ifdef H5_HAVE_PARALLEL
+ int mpi_rank = 0, mpi_size = 1;
+ int mpi_result;
+#endif /* H5_HAVE_PARALLEL */
FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL)
@@ -354,8 +358,51 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
/* Find the superblock */
- if(H5FD_locate_signature(&fdio_info, &super_addr) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+#ifdef H5_HAVE_PARALLEL
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
+
+ if((mpi_rank = H5F_mpi_get_rank(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Can't get MPI rank")
+
+ if((mpi_size = H5F_mpi_get_size(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size")
+ }
+
+ /* If we are an MPI application with at least two processes, the
+ * following superblock signature location optimization is applicable.
+ *
+ * Note:: For parallel applications which don't setup for using the
+ * HDF5 MPIO driver, we will arrive here with mpi_size == 1.
+ * This occurs because of the variable initialization (above) and the
+ * fact that we have skipped actually calling MPI functions to determine
+ * our MPI rank and size.
+ */
+ if ( mpi_size > 1 ) {
+ MPI_Comm this_comm = MPI_COMM_NULL;
+
+ if ( mpi_rank == 0 ) {
+ if (H5FD_locate_signature(&fdio_info, &super_addr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+ }
+ HDassert(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI));
+
+ if ( MPI_COMM_NULL == (this_comm = H5F_mpi_get_comm(f)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator")
+
+ if ( MPI_SUCCESS !=
+ (mpi_result = MPI_Bcast(&super_addr,sizeof(super_addr), MPI_BYTE, 0, this_comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result)
+ }
+ else {
+ /* Locate the signature as per per the serial library */
+#endif /* H5_HAVE_PARALLEL */
+
+ if (H5FD_locate_signature(&fdio_info, &super_addr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+
+#ifdef H5_HAVE_PARALLEL
+ }
+#endif /* H5_HAVE_PARALLEL */
if(HADDR_UNDEF == super_addr)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found")
@@ -406,12 +453,12 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock")
if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
- if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
- HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE")
+ if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE")
/* Enable all latest version support when file has v3 superblock */
if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3)
- f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
+ f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
/* Pin the superblock in the cache */
if(H5AC_pin_protected_entry(sblock) < 0)
@@ -511,15 +558,15 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
* been flushed to disk by the SWMR writer process.
*/
if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) {
- /*
- * When the file is opened for SWMR read access, skip the check if:
- * --the file is already marked for SWMR writing and
- * --the file has version 3 superblock for SWMR support
- */
- if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) &&
+ /*
+ * When the file is opened for SWMR read access, skip the check if:
+ * --the file is already marked for SWMR writing and
+ * --the file has version 3 superblock for SWMR support
+ */
+ if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) &&
(sblock->status_flags & H5F_SUPER_WRITE_ACCESS) &&
sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3)
- skip_eof_check = TRUE;
+ skip_eof_check = TRUE;
} /* end if */
if(!skip_eof_check && initial_read) {
if(HADDR_UNDEF == (eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT)))
@@ -593,7 +640,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
H5O_loc_t ext_loc; /* "Object location" for superblock extension */
H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */
H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */
- size_t u; /* Local index variable */
+ size_t u; /* Local index variable */
htri_t status; /* Status for message existing */
/* Sanity check - superblock extension should only be defined for
@@ -614,7 +661,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
} /* end if */
/* Open the superblock extension */
- if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
+ if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
/* Check for the extension having a 'driver info' message */
@@ -637,8 +684,8 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
/* Reset driver info message */
H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo);
- HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
- f->shared->drvinfo_sb_msg_exists = TRUE;
+ HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
+ f->shared->drvinfo_sb_msg_exists = TRUE;
} /* end else */
} /* end if */
@@ -764,37 +811,37 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
} /* end if not marked "unknown" */
} /* end if */
- /* Check for the extension having a 'metadata cache image' message */
+ /* Check for the extension having a 'metadata cache image' message */
if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
- hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0);
- H5O_mdci_t mdci_msg;
+ hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0);
+ H5O_mdci_t mdci_msg;
- /* if the metadata cache image superblock extension message exists,
+ /* if the metadata cache image superblock extension message exists,
* read its contents and pass the data on to the metadata cache.
* Given this data, the cache will load and decode the metadata
- * cache image block, decoded it and load its contents into the
- * the cache on the test protect call.
+ * cache image block, decoded it and load its contents into the
+ * the cache on the test protect call.
*
* Further, if the file is opened R/W, the metadata cache will
- * delete the metadata cache image superblock extension and free
- * the cache image block. Don't do this now as f->shared
- * is not fully setup, which complicates matters.
+ * delete the metadata cache image superblock extension and free
+ * the cache image block. Don't do this now as f->shared
+ * is not fully setup, which complicates matters.
*/
/* Retrieve the 'metadata cache image message' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id))
+ if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message")
/* Indicate to the cache that there's an image to load on first protect call */
if(H5AC_load_cache_image_on_next_protect(f, mdci_msg.addr, mdci_msg.size, rw) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed");
+ HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed");
} /* end if */
/* Close superblock extension */
if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
} /* end if */
/* Update the driver info if VFD indicated to do so */
@@ -974,6 +1021,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Bump superblock version if latest superblock version support is enabled */
if(H5F_USE_LATEST_FLAGS(f, H5F_LATEST_SUPERBLOCK))
super_vers = HDF5_SUPERBLOCK_VERSION_LATEST;
+ /* Bump superblock version to use version 3 superblock for SWMR writing */
+ else if((H5F_INTENT(f) & H5F_ACC_SWMR_WRITE))
+ super_vers = HDF5_SUPERBLOCK_VERSION_3;
/* Bump superblock version to create superblock extension for SOHM info */
else if(f->shared->sohm_nindexes > 0)
super_vers = HDF5_SUPERBLOCK_VERSION_2;
diff --git a/src/H5I.c b/src/H5I.c
index 42edf58..b8e47a2 100644
--- a/src/H5I.c
+++ b/src/H5I.c
@@ -451,9 +451,9 @@ H5I_nmembers(H5I_type_t type)
FUNC_ENTER_NOAPI(FAIL)
if(type <= H5I_BADID || type >= H5I_next_type)
- HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number")
if(NULL == (type_ptr = H5I_id_type_list_g[type]) || type_ptr->init_count <= 0)
- HGOTO_DONE(0);
+ HGOTO_DONE(0);
/* Set return value */
H5_CHECKED_ASSIGN(ret_value, int64_t, type_ptr->id_count, uint64_t);
diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h
index 7bc3c22..c737bbe 100644
--- a/src/H5Ipublic.h
+++ b/src/H5Ipublic.h
@@ -30,6 +30,8 @@
* When adding types here, add a section to the 'misc19' test in test/tmisc.c
* to verify that the H5I{inc|dec|get}_ref() routines work correctly with it.
*
+ * NOTE: H5I_REFERENCE is not used by the library and has been deprecated
+ * with a tentative removal version of 1.12.0. (DER, July 2017)
*/
typedef enum H5I_type_t {
H5I_UNINIT = (-2), /* uninitialized type */
@@ -40,12 +42,7 @@ typedef enum H5I_type_t {
H5I_DATASPACE, /* type ID for Dataspace objects */
H5I_DATASET, /* type ID for Dataset objects */
H5I_ATTR, /* type ID for Attribute objects */
- H5I_REFERENCE, /* type ID for Reference objects
- ** DEPRECATED**
- H5I_REFERENCE is not used in the library and
- will be removed from this enumeration in a
- future major release of the library.
- */
+ H5I_REFERENCE, /* *DEPRECATED* type ID for Reference objects */
H5I_VFL, /* type ID for virtual file layer */
H5I_GENPROP_CLS, /* type ID for generic property list classes */
H5I_GENPROP_LST, /* type ID for generic property lists */
diff --git a/src/H5MF.c b/src/H5MF.c
index d7af56a..49c7b77 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -115,6 +115,7 @@ hbool_t H5_PKG_INIT_VAR = FALSE;
/* Local Variables */
/*******************/
+
/*-------------------------------------------------------------------------
* Function: H5MF_init_merge_flags
diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c
index 1ae902f..0b300ba 100644
--- a/src/H5MFdbg.c
+++ b/src/H5MFdbg.c
@@ -155,8 +155,8 @@ done:
herr_t
H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int indent, int fwidth)
{
- herr_t ret_value = SUCCEED; /* Return value */
H5F_mem_page_t type; /* Memory type for iteration */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
diff --git a/src/H5Oefl.c b/src/H5Oefl.c
index 49c442f..ba7a6ee 100644
--- a/src/H5Oefl.c
+++ b/src/H5Oefl.c
@@ -151,7 +151,7 @@ H5O_efl_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh,
if((s = (const char *)H5HL_offset_into(heap, mesg->slot[u].name_offset)) == NULL)
HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "unable to get external file name")
- if(*s == (char)NULL)
+ if(*s == (char)'\0')
HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "invalid external file name")
mesg->slot[u].name = H5MM_xstrdup (s);
HDassert(mesg->slot[u].name);
diff --git a/src/H5Oflush.c b/src/H5Oflush.c
index 2d93221..9764f56 100644
--- a/src/H5Oflush.c
+++ b/src/H5Oflush.c
@@ -370,40 +370,40 @@ H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id, hbool_
type = H5I_get_type(oid);
switch(type) {
- case(H5I_GROUP):
+ case H5I_GROUP:
/* Re-open the group */
if(NULL == (object = H5G_open(obj_loc, dxpl_id)))
HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group")
break;
- case(H5I_DATATYPE):
+ case H5I_DATATYPE:
/* Re-open the named datatype */
if(NULL == (object = H5T_open(obj_loc, dxpl_id)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTOPENOBJ, FAIL, "unable to open named datatype")
break;
- case(H5I_DATASET):
+ case H5I_DATASET:
/* Re-open the dataset */
if(NULL == (object = H5D_open(obj_loc, H5P_DATASET_ACCESS_DEFAULT, dxpl_id)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset")
- if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */
- if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset")
+ if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */
+ if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset")
break;
- case(H5I_UNINIT):
- case(H5I_BADID):
- case(H5I_FILE):
- case(H5I_DATASPACE):
- case(H5I_ATTR):
- case(H5I_REFERENCE):
- case(H5I_VFL):
- case(H5I_GENPROP_CLS):
- case(H5I_GENPROP_LST):
- case(H5I_ERROR_CLASS):
- case(H5I_ERROR_MSG):
- case(H5I_ERROR_STACK):
- case(H5I_NTYPES):
+ case H5I_UNINIT:
+ case H5I_BADID:
+ case H5I_FILE:
+ case H5I_DATASPACE:
+ case H5I_ATTR:
+ case H5I_REFERENCE:
+ case H5I_VFL:
+ case H5I_GENPROP_CLS:
+ case H5I_GENPROP_LST:
+ case H5I_ERROR_CLASS:
+ case H5I_ERROR_MSG:
+ case H5I_ERROR_STACK:
+ case H5I_NTYPES:
default:
HGOTO_ERROR(H5E_ARGS, H5E_CANTRELEASE, FAIL, "not a valid file object ID (dataset, group, or datatype)")
break;
diff --git a/src/H5Otest.c b/src/H5Otest.c
index f0deade..68462a1 100644
--- a/src/H5Otest.c
+++ b/src/H5Otest.c
@@ -731,7 +731,7 @@ H5O_msg_move_to_new_chunk_test(hid_t oid, unsigned msg_type)
/* Allocate and initialize new chunk in the file, moving the found message */
/* (*new_idx returned from this routine is unused here) */
- if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, 40, oh->nmesgs, &found_msg, &new_idx) < 0)
+ if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, (curr_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh)), oh->nmesgs, &found_msg, &new_idx) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate new object header chunk")
/* Break out of loop, the message was found */
diff --git a/src/H5PB.c b/src/H5PB.c
index 52576f8..63e5e7a 100644
--- a/src/H5PB.c
+++ b/src/H5PB.c
@@ -1506,7 +1506,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry)
HDassert(page_entry);
/* Retrieve the 'eoa' for the file */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type)))
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, (H5FD_mem_t)page_entry->type)))
HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed")
/* If the starting address of the page is larger than
@@ -1525,7 +1525,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry)
fdio_info.meta_dxpl = fio_info->meta_dxpl;
fdio_info.raw_dxpl = fio_info->raw_dxpl;
- if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0)
+ if(H5FD_write(&fdio_info, (H5FD_mem_t)page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0)
HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed")
} /* end if */
diff --git a/src/H5PL.c b/src/H5PL.c
index 65d6c91..bd02bf3 100644
--- a/src/H5PL.c
+++ b/src/H5PL.c
@@ -22,135 +22,28 @@
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
-#include "H5MMprivate.h" /* Memory management */
#include "H5PLpkg.h" /* Plugin */
-#include "H5Zprivate.h" /* Filter pipeline */
/****************/
/* Local Macros */
/****************/
-#ifdef H5_HAVE_WIN32_API
-#define H5PL_EXPAND_ENV_VAR { \
- long bufCharCount; \
- char *tempbuf; \
- if(NULL == (tempbuf = (char *)H5MM_malloc(H5PL_EXPAND_BUFFER_SIZE))) \
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") \
- if((bufCharCount = ExpandEnvironmentStringsA(dl_path, tempbuf, H5PL_EXPAND_BUFFER_SIZE)) > H5PL_EXPAND_BUFFER_SIZE) { \
- tempbuf = (char *)H5MM_xfree(tempbuf); \
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") \
- } \
- if(bufCharCount == 0) { \
- tempbuf = (char *)H5MM_xfree(tempbuf); \
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") \
- } \
- dl_path = (char *)H5MM_xfree(dl_path); \
- dl_path = tempbuf; \
- }
-#else
-#define H5PL_EXPAND_ENV_VAR
-#endif /* H5_HAVE_WIN32_API */
-
-/****************************/
-/* Macros for supporting
- * both Windows and Unix */
-/****************************/
-/* Windows support
- *
- * SPECIAL WINDOWS NOTE
- *
- * Some of the Win32 API functions expand to fooA or fooW depending on
- * whether UNICODE or _UNICODE are defined. You MUST explicitly use
- * the A version of the functions to force char * behavior until we
- * work out a scheme for proper Windows Unicode support.
- *
- * If you do not do this, people will be unable to incorporate our
- * source code into their own CMake builds if they define UNICODE.
- */
-#ifdef H5_HAVE_WIN32_API
-
-#define H5PL_PATH_SEPARATOR ";"
-
-/* Handle for dynamic library */
-#define H5PL_HANDLE HINSTANCE
-
-/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
-#define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)
-
-/* Get the address of a symbol in dynamic library */
-#define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N)
-
-/* Close dynamic library */
-#define H5PL_CLOSE_LIB(H) FreeLibrary(H)
-
-/* Clear error - nothing to do */
-#define H5PL_CLR_ERROR
-
-/* maximum size for expanding env vars */
-#define H5PL_EXPAND_BUFFER_SIZE 32767
-
-typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void);
-
-/* Unix support */
-#else /* H5_HAVE_WIN32_API */
-
-#define H5PL_PATH_SEPARATOR ":"
-
-/* Handle for dynamic library */
-#define H5PL_HANDLE void *
-
-/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
-#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY)
-
-/* Get the address of a symbol in dynamic library */
-#define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N)
-
-/* Close dynamic library */
-#define H5PL_CLOSE_LIB(H) dlclose(H)
-
-/* Clear error */
-#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror())
-
-typedef const void *(*H5PL_get_plugin_info_t)(void);
-#endif /* H5_HAVE_WIN32_API */
-
-/* Whether to preload pathnames for plugin libraries */
-#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR
-
-/* Special symbol to indicate no plugin loading */
-#define H5PL_NO_PLUGIN "::"
/******************/
/* Local Typedefs */
/******************/
-/* Type for the list of info for opened plugin libraries */
-typedef struct H5PL_table_t {
- H5PL_type_t pl_type; /* plugin type */
- int pl_id; /* ID for the plugin */
- H5PL_HANDLE handle; /* plugin handle */
-} H5PL_table_t;
-
/********************/
/* Local Prototypes */
/********************/
-static herr_t H5PL__init_path_table(void);
-static htri_t H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info);
-static htri_t H5PL__open(H5PL_type_t pl_type, char *libname, int plugin_id, const void **pl_info);
-static htri_t H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info);
-static herr_t H5PL__close(H5PL_HANDLE handle);
-
/*********************/
/* Package Variables */
/*********************/
-/* Package initialization variable */
-hbool_t H5_PKG_INIT_VAR = FALSE;
-
/*****************************/
/* Library Private Variables */
@@ -161,145 +54,42 @@ hbool_t H5_PKG_INIT_VAR = FALSE;
/* Local Variables */
/*******************/
-/* Table for opened plugin libraries */
-static size_t H5PL_table_alloc_g = 0;
-static size_t H5PL_table_used_g = 0;
-static H5PL_table_t *H5PL_table_g = NULL;
-
-/* Table of location paths for plugin libraries */
-static char *H5PL_path_table_g[H5PL_MAX_PATH_NUM];
-static size_t H5PL_num_paths_g = 0;
-static hbool_t H5PL_path_found_g = FALSE;
-
-/* Enable all plugin libraries */
-static unsigned int H5PL_plugin_g = H5PL_ALL_PLUGIN;
-
-
-
-/*--------------------------------------------------------------------------
-NAME
- H5PL__init_package -- Initialize interface-specific information
-USAGE
- herr_t H5PL__init_package()
-RETURNS
- Non-negative on success/Negative on failure
-DESCRIPTION
- Initializes any interface-specific data or routines.
-
---------------------------------------------------------------------------*/
-herr_t
-H5PL__init_package(void)
-{
- char *preload_path;
-
- FUNC_ENTER_PACKAGE_NOERR
-
- /* Retrieve pathnames from HDF5_PLUGIN_PRELOAD if the user sets it
- * to tell the library to load plugin libraries without search.
- */
- if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD")))
- /* Special symbol "::" means no plugin during data reading. */
- if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN))
- H5PL_plugin_g = 0;
-
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5PL__init_package() */
-
/*-------------------------------------------------------------------------
- * Function: H5PL_term_package
+ * Function: H5PLset_loading_state
*
- * Purpose: Terminate the H5PL interface: release all memory, reset all
- * global variables to initial values. This only happens if all
- * types have been destroyed from other interfaces.
- *
- * Return: Success: Positive if any action was taken that might
- * affect some other interface; zero otherwise.
- * Failure: Negative.
- *
- * Programmer: Raymond Lu
- * 20 February 2013
- *
- *-------------------------------------------------------------------------
- */
-int
-H5PL_term_package(void)
-{
- int n = 0;
-
- FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- if(H5_PKG_INIT_VAR) {
- size_t u; /* Local index variable */
-
- /* Close opened dynamic libraries */
- if(H5PL_table_g) {
- for(u = 0; u < H5PL_table_used_g; u++)
- H5PL__close((H5PL_table_g[u]).handle);
-
- /* Free the table of dynamic libraries */
- H5PL_table_g = (H5PL_table_t *)H5MM_xfree(H5PL_table_g);
- H5PL_table_used_g = H5PL_table_alloc_g = 0;
-
- n++;
- } /* end if */
-
- /* Free the table of search paths */
- if(H5PL_num_paths_g > 0) {
- for(u = 0; u < H5PL_num_paths_g; u++)
- if(H5PL_path_table_g[u])
- H5PL_path_table_g[u] = (char *)H5MM_xfree(H5PL_path_table_g[u]);
- H5PL_num_paths_g = 0;
- H5PL_path_found_g = FALSE;
-
- n++;
- } /* end if */
-
- /* Mark the interface as uninitialized */
- if(0 == n)
- H5_PKG_INIT_VAR = FALSE;
- } /* end if */
-
- FUNC_LEAVE_NOAPI(n)
-} /* end H5PL_term_package() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PLset_loading_state
+ * Purpose: Control the loading of dynamic plugin types.
*
- * Purpose: Control the loading of dynamic plugin types.
+ * The plugin_control_mask parameter is a bitfield that controls
+ * whether certain classes of plugins (e.g.: filters,
+ * VOL drivers) will be loaded by the library.
*
- * This function will not allow plugin types if the pathname from the HDF5_PLUGIN_PRELOAD
- * environment variable is set to the special "::" string.
+ * plugin bit = 0, will prevent the use of that dynamic plugin type.
+ * plugin bit = 1, will allow the use of that dynamic plugin type.
*
- * plugin bit = 0, will prevent the use of that dynamic plugin type.
- * plugin bit = 1, will allow the use of that dynamic plugin type.
+ * A list of pre-defined masks can be found in H5PLpublic.h.
+ * Set the mask to 0 to disable all plugins.
*
- * H5PL_TYPE_FILTER changes just dynamic filters
- * A H5PL_ALL_PLUGIN will enable all dynamic plugin types
- * A zero value will disable all dynamic plugin types
+ * This function will not allow plugin types if the pathname
+ * from the HDF5_PLUGIN_PRELOAD environment variable is set to
+ * the special "::" string.
*
- * Return: Non-negative or success
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLset_loading_state(unsigned int plugin_type)
+H5PLset_loading_state(unsigned int plugin_control_mask)
{
- char *preload_path;
- herr_t ret_value = SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "Iu", plugin_type);
+ H5TRACE1("e", "Iu", plugin_control_mask);
- /* change the bit value of the requested plugin type(s) */
- H5PL_plugin_g = plugin_type;
-
- /* check if special ENV variable is set and disable all plugin types */
- if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD")))
- /* Special symbol "::" means no plugin during data reading. */
- if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN))
- H5PL_plugin_g = 0;
+ /* Set the plugin control mask */
+ if(H5PL__set_plugin_control_mask(plugin_control_mask) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTSET, FAIL, "error setting plugin control mask")
done:
FUNC_LEAVE_API(ret_value)
@@ -307,27 +97,35 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5PLget_loading_state
+ * Function: H5PLget_loading_state
*
- * Purpose: Query state of the loading of dynamic plugin types.
+ * Purpose: Get the bitmask that controls whether certain classes
+ * of plugins (e.g.: filters, VOL drivers) will be loaded
+ * by the library.
*
- * This function will return the state of the global flag.
+ * Zero if all plugin types are disabled
+ * Negative if all plugin types are enabled
+ * Positive if one or more of the plugin types are enabled
*
- * Return: Zero if all plugin types are disabled, negative if all
- * plugin types are enabled, positive if one or more of the plugin types are enabled.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLget_loading_state(unsigned int *plugin_type)
+H5PLget_loading_state(unsigned int *plugin_control_mask)
{
- herr_t ret_value = SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*Iu", plugin_type);
+ H5TRACE1("e", "*Iu", plugin_control_mask);
- if(plugin_type)
- *plugin_type = H5PL_plugin_g;
+ if (NULL == plugin_control_mask)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_control_mask parameter cannot be NULL")
+
+ /* Set the plugin control mask */
+ if(H5PL__get_plugin_control_mask(plugin_control_mask) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error getting plugin control mask")
done:
FUNC_LEAVE_API(ret_value)
@@ -335,674 +133,282 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5PL_load
- *
- * Purpose: Given the plugin type and identifier, this function searches
- * and/or loads a dynamic plugin library first among the already
- * opened libraries then in the designated location paths.
- *
- * Return: Non-NULL on success/NULL on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-const void *
-H5PL_load(H5PL_type_t type, int id)
-{
- htri_t found; /* Whether the plugin was found */
- const void *plugin_info = NULL;
- const void *ret_value = NULL;
-
- FUNC_ENTER_NOAPI(NULL)
-
- switch(type) {
- case H5PL_TYPE_FILTER:
- if((H5PL_plugin_g & H5PL_FILTER_PLUGIN) == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id)
- break;
-
- case H5PL_TYPE_ERROR:
- case H5PL_TYPE_NONE:
- default:
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id)
- } /* end switch */
-
- /* Initialize the location paths for dynamic libraries, if they aren't
- * already set up.
- */
- if(FALSE == H5PL_path_found_g)
- if(H5PL__init_path_table() < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, NULL, "can't initialize search path table")
-
- /* Search in the table of already loaded plugin libraries */
- if((found = H5PL__search_table(type, id, &plugin_info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in table failed")
-
- /* If not found, iterate through the path table to find the right dynamic library */
- if(!found) {
- size_t i; /* Local index variable */
-
- for(i = 0; i < H5PL_num_paths_g; i++) {
- if((found = H5PL__find(type, id, H5PL_path_table_g[i], &plugin_info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in paths failed")
-
- /* Break out if found */
- if(found) {
- HDassert(plugin_info);
- break;
- } /* end if */
- } /* end for */
- } /* end if */
-
- /* Check if we found the plugin */
- if(found)
- ret_value = plugin_info;
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL_load() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PLappend
+ * Function: H5PLappend
*
- * Purpose: Insert a plugin path at the end of the list.
+ * Purpose: Insert a plugin search path at the end of the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLappend(const char *plugin_path)
+H5PLappend(const char *search_path)
{
herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*s", plugin_path);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+ H5TRACE1("e", "*s", search_path);
- H5PL_EXPAND_ENV_VAR
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- H5PL_path_table_g[H5PL_num_paths_g] = dl_path;
- H5PL_num_paths_g++;
+ /* Append the search path to the path table */
+ if (H5PL__append_path(search_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTAPPEND, FAIL, "unable to append search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLappend() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLprepend
+ * Function: H5PLprepend
*
- * Purpose: Insert a plugin path at the beginning of the list.
+ * Purpose: Insert a plugin search path at the beginning of the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLprepend(const char *plugin_path)
+H5PLprepend(const char *search_path)
{
herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
- unsigned int plindex;
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*s", plugin_path);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+ H5TRACE1("e", "*s", search_path);
- H5PL_EXPAND_ENV_VAR
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- for (plindex = (unsigned int)H5PL_num_paths_g; plindex > 0; plindex--)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1];
- H5PL_path_table_g[0] = dl_path;
- H5PL_num_paths_g++;
+ /* Prepend the search path to the path table */
+ if (H5PL__prepend_path(search_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLprepend() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLreplace
+ * Function: H5PLreplace
*
- * Purpose: Replace the path at the specified index.
+ * Purpose: Replace the path at the specified index. The path at the
+ * index must exist.
*
- * Return: Non-negative or success.
+ * Return: Non-negative or success.
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLreplace(const char *plugin_path, unsigned int index)
+H5PLreplace(const char *search_path, unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE2("e", "*sIu", plugin_path, index);
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+ H5TRACE2("e", "*sIu", search_path, index);
+
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- H5PL_EXPAND_ENV_VAR
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
- if(H5PL_path_table_g[index])
- H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]);
- H5PL_path_table_g[index] = dl_path;
+ /* Insert the search path into the path table */
+ if (H5PL__replace_path(search_path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLreplace() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLinsert
+ * Function: H5PLinsert
*
- * Purpose: Insert a plugin path at the specified index, moving other paths after the index.
+ * Purpose: Insert a plugin search path at the specified index, moving
+ * other paths after the index.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLinsert(const char *plugin_path, unsigned int index)
+H5PLinsert(const char *search_path, unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
- unsigned int plindex;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE2("e", "*sIu", plugin_path, index);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
-
- H5PL_EXPAND_ENV_VAR
-
- for(plindex = (unsigned int)H5PL_num_paths_g; plindex > index; plindex--)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1];
- H5PL_path_table_g[index] = dl_path;
- H5PL_num_paths_g++;
+ H5TRACE2("e", "*sIu", search_path, index);
+
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if ((0 != num_paths) && (index >= num_paths))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Insert the search path into the path table */
+ if (H5PL__insert_path(search_path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLinsert() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLremove
+ * Function: H5PLremove
*
- * Purpose: Remove the plugin path at the specifed index and compacting the list.
+ * Purpose: Remove the plugin path at the specifed index and compact
+ * the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
+ *
+ * Return: Non-negative or success.
*
*-------------------------------------------------------------------------
*/
herr_t
H5PLremove(unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- unsigned int plindex;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE1("e", "Iu", index);
- if(H5PL_num_paths_g == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == H5PL_path_table_g[index])
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index")
- H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]);
-
- H5PL_num_paths_g--;
- for(plindex = index; plindex < (unsigned int)H5PL_num_paths_g; plindex++)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex + 1];
- H5PL_path_table_g[H5PL_num_paths_g] = NULL;
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Delete the search path from the path table */
+ if (H5PL__remove_path(index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "unable to remove search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLremove() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLget
+ * Function: H5PLget
+ *
+ * Purpose: Query the plugin path at a specified index.
*
- * Purpose: Query the plugin path at the specified index.
+ * If 'path_buf' is non-NULL then up to 'buf_size' bytes will be written into
+ * that buffer and the length of the path name will be returned.
*
- * Return: Success: The length of path.
+ * If 'path_buf' is NULL, this function will simply return the number of
+ * characters required to store the path name, ignoring 'path_buf' and
+ * 'buf_size'
*
- * If `pathname' is non-NULL then write up to `size' bytes into that
- * buffer and always return the length of the pathname.
- * Otherwise `size' is ignored and the function does not store the pathname,
- * just returning the number of characters required to store the pathname.
- * If an error occurs then the buffer pointed to by `pathname' (NULL or non-NULL)
- * is unchanged and the function returns a negative value.
- * If a zero is returned for the name's length, then there is no pathname
- * associated with the index.
+ * If an error occurs then the buffer pointed to by 'path_buf'
+ * (NULL or non-NULL) will be unchanged and the function will return a
+ * negative value.
+ *
+ * If a zero is returned for the name's length, then there is no path name
+ * associated with the index and the 'path_buf' buffer will be unchanged.
+ *
+ * Return: Success: The length of path
+ * Failure: A negative value
*
*-------------------------------------------------------------------------
*/
ssize_t
-H5PLget(unsigned int index, char *pathname/*out*/, size_t size)
+H5PLget(unsigned int index, char *path_buf, size_t buf_size)
{
- ssize_t ret_value = 0; /* Return value */
- size_t len = 0; /* Length of pathname */
- char *dl_path = NULL;
+ unsigned num_paths; /* Current number of stored paths */
+ const char *path = NULL; /* path from table */
+ size_t path_len = 0; /* Length of path */
+ ssize_t ret_value = 0; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE3("Zs", "Iuxz", index, pathname, size);
- if(H5PL_num_paths_g == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5PL_path_table_g[index]))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index")
- len = HDstrlen(dl_path);
- if(pathname) {
- HDstrncpy(pathname, dl_path, MIN((size_t)(len + 1), size));
- if((size_t)len >= size)
- pathname[size - 1] = '\0';
+ H5TRACE3("Zs", "Iu*sz", index, path_buf, buf_size);
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Check if the search table is empty */
+ if (H5PL__get_num_paths() == 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, (-1), "plugin search path table is empty")
+
+ /* Get the path at the specified index and its length */
+ if (NULL == (path = H5PL__get_path(index)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, (-1), "no path stored at that index")
+ path_len = HDstrlen(path);
+
+ /* If the path buffer is not NULL, copy the path to the buffer */
+ if (path_buf) {
+ HDstrncpy(path_buf, path, MIN((size_t)(path_len + 1), buf_size));
+ if ((size_t)path_len >= buf_size)
+ path_buf[buf_size - 1] = '\0';
} /* end if */
/* Set return value */
- ret_value = (ssize_t)len;
+ ret_value = (ssize_t)path_len;
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLget() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLsize
+ * Function: H5PLsize
*
- * Purpose: Query the size of the current list of plugin paths.
+ * Purpose: Get the number of stored plugin paths.
+ * XXX: This is a terrible name. Can it be changed?
*
- * Return: Plugin path size
+ * Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLsize(unsigned int *listsize)
+H5PLsize(unsigned int *num_paths)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*Iu", listsize);
+ H5TRACE1("e", "*Iu", num_paths);
+
+ /* Check arguments */
+ if (!num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "num_paths parameter cannot be NULL")
- *listsize = (unsigned int)H5PL_num_paths_g;
+ /* Get the number of stored plugin paths */
+ *num_paths = H5PL__get_num_paths();
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLsize() */
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__init_path_table
- *
- * Purpose: Initialize the path table.
- *
- * Return: Non-negative on success/Negative on failure
- *
- * Programmer: Quincey Koziol
- * 18 March 2013
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5PL__init_path_table(void)
-{
- char *dl_path = NULL;
- char *origin_dl_path;
- char *dir;
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_STATIC
-
- /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it
- * or from the default paths if it isn't set.
- */
- origin_dl_path = HDgetenv("HDF5_PLUGIN_PATH");
- if(NULL == origin_dl_path)
- dl_path = H5MM_strdup(H5PL_DEFAULT_PATH);
- else
- dl_path = H5MM_strdup(origin_dl_path);
- if(NULL == dl_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
-
- H5PL_EXPAND_ENV_VAR
-
- /* Put paths in the path table. They are separated by ":" */
- dir = HDstrtok(dl_path, H5PL_PATH_SEPARATOR);
- while(dir) {
- /* Check for too many directories in path */
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == (H5PL_path_table_g[H5PL_num_paths_g] = H5MM_strdup(dir)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- H5PL_num_paths_g++;
- dir = HDstrtok(NULL, H5PL_PATH_SEPARATOR);
- } /* end while */
-
- H5PL_path_found_g = TRUE;
-
-done:
- if(dl_path)
- dl_path = (char *)H5MM_xfree(dl_path);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__init_path_table() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__find
- *
- * Purpose: Given a path, this function opens the directory and envokes
- * another function to go through all files to find the right
- * plugin library. Two function definitions are for Unix and
- * Windows.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-#ifndef H5_HAVE_WIN32_API
-static htri_t
-H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info)
-{
- char *pathname = NULL;
- DIR *dirp = NULL;
- struct dirent *dp;
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Open the directory */
- if(!(dirp = HDopendir(dir)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir)
-
- /* Iterates through all entries in the directory to find the right plugin library */
- while(NULL != (dp = HDreaddir(dirp))) {
- /* The library we are looking for should be called libxxx.so... on Unix
- * or libxxx.xxx.dylib on Mac.
- */
-#ifndef __CYGWIN__
- if(!HDstrncmp(dp->d_name, "lib", (size_t)3) &&
- (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) {
-#else
- if(!HDstrncmp(dp->d_name, "cyg", (size_t)3) &&
- HDstrstr(dp->d_name, ".dll") ) {
-
-#endif
- h5_stat_t my_stat;
- size_t pathname_len;
- htri_t found_in_dir;
-
- /* Allocate & initialize the path name */
- pathname_len = HDstrlen(dir) + HDstrlen(dp->d_name) + 2;
- if(NULL == (pathname = (char *)H5MM_malloc(pathname_len)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- HDsnprintf(pathname, pathname_len, "%s/%s", dir, dp->d_name);
-
- /* Get info for directory entry */
- if(HDstat(pathname, &my_stat) == -1)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file: %s", HDstrerror(errno))
-
- /* If it is a directory, skip it */
- if(S_ISDIR(my_stat.st_mode))
- continue;
-
- /* Attempt to open the dynamic library as a filter library */
- if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
- if(found_in_dir)
- HGOTO_DONE(TRUE) /* Indicate success */
- pathname = (char *)H5MM_xfree(pathname);
- } /* end if */
- } /* end while */
-
-done:
- if(dirp)
- if(HDclosedir(dirp) < 0)
- HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno))
- pathname = (char *)H5MM_xfree(pathname);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__find() */
-#else /* H5_HAVE_WIN32_API */
-static htri_t
-H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info)
-{
- WIN32_FIND_DATAA fdFile;
- HANDLE hFind;
- char *pathname = NULL;
- char service[2048];
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Specify a file mask. *.* = We want everything! */
- sprintf(service, "%s\\*.dll", dir);
- if((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE)
- HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory")
-
- do {
- /* Find first file will always return "."
- * and ".." as the first two directories.
- */
- if(HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) {
- size_t pathname_len;
- htri_t found_in_dir;
-
- /* Allocate & initialize the path name */
- pathname_len = HDstrlen(dir) + HDstrlen(fdFile.cFileName) + 2;
- if(NULL == (pathname = (char *)H5MM_malloc(pathname_len)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- HDsnprintf(pathname, pathname_len, "%s\\%s", dir, fdFile.cFileName);
-
- /* Is the entity a File or Folder? */
- if(fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
- continue;
-
- if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
- if(found_in_dir)
- HGOTO_DONE(TRUE) /* Indicate success */
- pathname = (char *)H5MM_xfree(pathname);
- } /* end if */
- } while(FindNextFileA(hFind, &fdFile)); /* Find the next file. */
-
-done:
- if(hFind)
- FindClose(hFind);
- if(pathname)
- pathname = (char *)H5MM_xfree(pathname);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__find() */
-#endif /* H5_HAVE_WIN32_API */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__open
- *
- * Purpose: Iterates through all files to find the right plugin library.
- * It loads the dynamic plugin library and keeps it on the list
- * of loaded libraries.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static htri_t
-H5PL__open(H5PL_type_t pl_type, char *libname, int pl_id, const void **pl_info)
-{
- H5PL_HANDLE handle = NULL;
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* There are different reasons why a library can't be open, e.g. wrong architecture.
- * simply continue if we can't open it.
- */
- if(NULL == (handle = H5PL_OPEN_DLIB(libname))) {
- H5PL_CLR_ERROR; /* clear error */
- } /* end if */
- else {
- H5PL_get_plugin_info_t get_plugin_info = NULL;
-
- /* Return a handle for the function H5PLget_plugin_info in the dynamic library.
- * The plugin library is suppose to define this function.
- */
- if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) {
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- } /* end if */
- else {
- const H5Z_class2_t *plugin_info;
-
- /* Invoke H5PLget_plugin_info to verify this is the right library we are looking for.
- * Move on if it isn't.
- */
- if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) {
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info")
- } /* end if */
-
- /* Successfully found plugin library, check if it's the right one */
- if(plugin_info->id == pl_id) {
- /* Expand the table if it is too small */
- if(H5PL_table_used_g >= H5PL_table_alloc_g) {
- size_t n = MAX(H5Z_MAX_NFILTERS, 2 * H5PL_table_alloc_g);
- H5PL_table_t *table = (H5PL_table_t *)H5MM_realloc(H5PL_table_g, n * sizeof(H5PL_table_t));
-
- if(!table)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend dynamic library table")
-
- H5PL_table_g = table;
- H5PL_table_alloc_g = n;
- } /* end if */
-
- (H5PL_table_g[H5PL_table_used_g]).handle = handle;
- (H5PL_table_g[H5PL_table_used_g]).pl_type = pl_type;
- (H5PL_table_g[H5PL_table_used_g]).pl_id = plugin_info->id;
- H5PL_table_used_g++;
-
- /* Set the plugin info to return */
- *pl_info = (const void *)plugin_info;
-
- /* Indicate success */
- ret_value = TRUE;
- } /* end if */
- else
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- } /* end if */
- } /* end else */
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__open() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__search_table
- *
- * Purpose: Search in the list of already opened dynamic libraries
- * to see if the one we are looking for is already opened.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * Negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static htri_t
-H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info)
-{
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Search in the table of already opened dynamic libraries */
- if(H5PL_table_used_g > 0) {
- size_t i;
-
- for(i = 0; i < H5PL_table_used_g; i++) {
- if((plugin_type == (H5PL_table_g[i]).pl_type) && (type_id == (H5PL_table_g[i]).pl_id)) {
- H5PL_get_plugin_info_t get_plugin_info;
- const H5Z_class2_t *plugin_info;
-
- if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_table_g[i]).handle, "H5PLget_plugin_info")))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info")
-
- if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)()))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info")
-
- *info = plugin_info;
- HGOTO_DONE(TRUE)
- } /* end if */
- } /* end for */
- } /* end if */
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__search_table() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__close
- *
- * Purpose: Closes the handle for dynamic library
- *
- * Return: Non-negative on success/Negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5PL__close(H5PL_HANDLE handle)
-{
- FUNC_ENTER_STATIC_NOERR
-
- H5PL_CLOSE_LIB(handle);
-
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5PL__close() */
-
diff --git a/src/H5PLint.c b/src/H5PLint.c
new file mode 100644
index 0000000..c887f86
--- /dev/null
+++ b/src/H5PLint.c
@@ -0,0 +1,384 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Internal routines for managing plugins.
+ *
+ */
+
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+#include "H5Zprivate.h" /* Filter pipeline */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Package initialization variable */
+hbool_t H5_PKG_INIT_VAR = FALSE;
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Bitmask that controls whether classes of plugins
+ * (e.g.: filters, VOL drivers) can be loaded.
+ */
+static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN;
+
+/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD
+ * environment variable was set to H5PL_NO_PLUGIN at
+ * package initialization.
+ */
+static hbool_t H5PL_allow_plugins_g = TRUE;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_plugin_control_mask
+ *
+ * Purpose: Gets the internal plugin control mask value.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__get_plugin_control_mask(unsigned int *mask /*out*/)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Check args - Just assert on package functions */
+ HDassert(mask);
+
+ /* Return the mask */
+ *mask = H5PL_plugin_control_mask_g;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__get_plugin_control_mask() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__set_plugin_control_mask
+ *
+ * Purpose: Sets the internal plugin control mask value.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__set_plugin_control_mask(unsigned int mask)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Only allow setting this if plugins have not been disabled.
+ * XXX: Note that we don't consider this an error, but instead
+ * silently ignore it. We may want to consider this behavior
+ * more carefully.
+ */
+ if (H5PL_allow_plugins_g)
+ H5PL_plugin_control_mask_g = mask;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__set_plugin_control_mask() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__init_package
+ *
+ * Purpose: Initialize any package-specific data and call any init
+ * routines for the package.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__init_package(void)
+{
+ char *env_var = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check the environment variable to determine if the user wants
+ * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in
+ * H5PLpublic.h) means we don't want to load plugins.
+ */
+ if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD")))
+ if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) {
+ H5PL_plugin_control_mask_g = 0;
+ H5PL_allow_plugins_g = FALSE;
+ }
+
+ /* Create the table of previously-loaded plugins */
+ if (H5PL__create_plugin_cache() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache")
+
+ /* Create the table of search paths for dynamic libraries */
+ if (H5PL__create_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__init_package() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL_term_package
+ *
+ * Purpose: Terminate the H5PL interface: release all memory, reset all
+ * global variables to initial values. This only happens if all
+ * types have been destroyed from other interfaces.
+ *
+ * Return: Success: Positive if any action was taken that might
+ * affect some other interface; zero otherwise
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+H5PL_term_package(void)
+{
+ hbool_t already_closed = FALSE;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (H5_PKG_INIT_VAR) {
+
+ /* Close the plugin cache.
+ * We need to bump the return value if we did any real work here.
+ */
+ if (H5PL__close_plugin_cache(&already_closed) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache")
+ if (!already_closed)
+ ret_value++;
+
+ /* Close the search path table and free the paths */
+ if (H5PL__close_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table")
+
+ /* Mark the interface as uninitialized */
+ if (0 == ret_value)
+ H5_PKG_INIT_VAR = FALSE;
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL_term_package() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL_load
+ *
+ * Purpose: Given the plugin type and identifier, this function searches
+ * for and, if found, loads a dynamic plugin library.
+ *
+ * The function searches first in the cached plugins and then
+ * in the paths listed in the path table.
+ *
+ * Return: Success: A pointer to the plugin info
+ * Failure: NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+const void *
+H5PL_load(H5PL_type_t type, int id)
+{
+ H5PL_search_params_t search_params;
+ hbool_t found = FALSE; /* Whether the plugin was found */
+ const void *plugin_info = NULL;
+ const void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI(NULL)
+
+ /* Check if plugins can be loaded for this plugin type */
+ switch (type) {
+ case H5PL_TYPE_FILTER:
+ if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id)
+ break;
+
+ case H5PL_TYPE_ERROR:
+ case H5PL_TYPE_NONE:
+ default:
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id)
+ }
+
+ /* Set up the search parameters */
+ search_params.type = type;
+ search_params.id = id;
+
+ /* Search in the table of already loaded plugin libraries */
+ if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed")
+
+ /* If not found, try iterating through the path table to find an appropriate plugin */
+ if (!found)
+ if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed")
+
+ /* Set the return value we found the plugin */
+ if (found)
+ ret_value = plugin_info;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL_load() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__open
+ *
+ * Purpose: Opens a plugin.
+ *
+ * The success parameter will be set to TRUE and the plugin_info
+ * parameter will be filled in on success. Otherwise, they
+ * will be FALSE and NULL, respectively.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting
+ * object pointers to function pointers, which is undefined in ANSI C.
+ * This is basically unavoidable due to the nature of dlsym() and *is*
+ * defined in POSIX, so it's fine.
+ *
+ * This pragma only needs to surround the assignment of the
+ * get_plugin_info function pointer, but early (4.4.7, at least) gcc
+ * only allows diagnostic pragmas to be toggled outside of functions.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+herr_t
+H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info)
+{
+ H5PL_HANDLE handle = NULL;
+ H5PL_get_plugin_info_t get_plugin_info = NULL;
+ htri_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(success);
+ HDassert(plugin_info);
+
+ /* Initialize out parameters */
+ *success = FALSE;
+ *plugin_info = NULL;
+
+ /* There are different reasons why a library can't be open, e.g. wrong architecture.
+ * If we can't open the library, just return.
+ */
+ if (NULL == (handle = H5PL_OPEN_DLIB(path))) {
+ H5PL_CLR_ERROR; /* clear error */
+ HGOTO_DONE(SUCCEED);
+ }
+
+ /* Return a handle for the function H5PLget_plugin_info in the dynamic library.
+ * The plugin library is suppose to define this function.
+ */
+ if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) {
+
+ const H5Z_class2_t *info;
+
+ /* Get the plugin info */
+ if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)()))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info")
+
+ /* Check if the filter IDs match */
+ if (info->id == id) {
+
+ /* Store the plugin in the cache */
+ if (H5PL__add_plugin(type, id, handle))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache")
+
+ /* Set output parameters */
+ *success = TRUE;
+ *plugin_info = (const void *)info;
+ }
+ }
+
+done:
+ if (!success && handle)
+ if (H5PL__close(handle) < 0)
+ HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__open() */
+#pragma GCC diagnostic pop
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close
+ *
+ * Purpose: Closes the handle for dynamic library
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close(H5PL_HANDLE handle)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ H5PL_CLOSE_LIB(handle);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5PL__close() */
+
diff --git a/src/H5PLpath.c b/src/H5PLpath.c
new file mode 100644
index 0000000..435802a
--- /dev/null
+++ b/src/H5PLpath.c
@@ -0,0 +1,776 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Code to implement a path table which stores plugin search paths.
+ *
+ * The path table is implemented as a dynamic, global array which
+ * will grow as new paths are inserted. The capacity of the path
+ * table never shrinks (though given the low number of paths
+ * expected and the low likelihood of paths being removed, this
+ * seems unlikely to be a problem). Inserts and removals rework
+ * the array so that there are no 'holes' in the in-use part
+ * of the array.
+ *
+ * Note that it's basically up to the user to manage the indexes
+ * when a complicated series of insert, overwrite, and, remove
+ * operations take place.
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+/* Initial capacity of the path table */
+#define H5PL_INITIAL_PATH_CAPACITY 16
+
+/* The amount to add to the capacity when the table is full */
+#define H5PL_PATH_CAPACITY_ADD 16
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static herr_t H5PL__insert_at(const char *path, unsigned int index);
+static herr_t H5PL__make_space_at(unsigned int index);
+static herr_t H5PL__replace_at(const char *path, unsigned int index);
+static herr_t H5PL__expand_path_table(void);
+static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info);
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Stored plugin paths to search */
+static char **H5PL_paths_g = NULL;
+
+/* The number of stored paths */
+static unsigned H5PL_num_paths_g = 0;
+
+/* The capacity of the path table */
+static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__insert_at()
+ *
+ * Purpose: Insert a path at a particular index in the path table.
+ * Does not clobber! Will move existing paths up to make
+ * room. Use H5PL__replace_at(index) if you want to clobber.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__insert_at(const char *path, unsigned int index)
+{
+ char *path_copy = NULL; /* copy of path string (for storing) */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Expand the table if it is full */
+ if (H5PL_num_paths_g == H5PL_path_capacity_g)
+ if (H5PL__expand_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table")
+
+ /* Copy the path for storage so the caller can dispose of theirs */
+ if (NULL == (path_copy = H5MM_strdup(path)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
+
+#ifdef H5_HAVE_WIN32_API
+ /* Clean up Microsoft Windows environment variables in the path string */
+ if(H5_expand_windows_env_vars(&path_copy))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
+#endif /* H5_HAVE_WIN32_API */
+
+ /* If the table entry is in use, make some space */
+ if (H5PL_paths_g[index])
+ if (H5PL__make_space_at(index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry")
+
+ /* Insert the copy of the search path into the table at the specified index */
+ H5PL_paths_g[index] = path_copy;
+ H5PL_num_paths_g++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__insert_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__make_space_at()
+ *
+ * Purpose: Free up a slot in the path table, moving existing path
+ * entries as necessary.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__make_space_at(unsigned int index)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check args - Just assert on package functions */
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Copy the paths back to make a space */
+ for (u = H5PL_num_paths_g; u > index; u--)
+ H5PL_paths_g[u] = H5PL_paths_g[u-1];
+
+ H5PL_paths_g[index] = NULL;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__make_space_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__replace_at()
+ *
+ * Purpose: Replace a path at a particular index in the path table.
+ * The path in the table must exist and will be freed by this
+ * function.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__replace_at(const char *path, unsigned int index)
+{
+ char *path_copy = NULL; /* copy of path string (for storing) */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Check that the table entry is in use */
+ if (!H5PL_paths_g[index])
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index)
+
+ /* Copy the path for storage so the caller can dispose of theirs */
+ if (NULL == (path_copy = H5MM_strdup(path)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
+
+#ifdef H5_HAVE_WIN32_API
+ /* Clean up Microsoft Windows environment variables in the path string */
+ if (H5_expand_windows_env_vars(&path_copy))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
+#endif /* H5_HAVE_WIN32_API */
+
+ /* Free the existing path entry */
+ H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
+
+ /* Copy the search path into the table at the specified index */
+ H5PL_paths_g[index] = path_copy;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__create_path_table
+ *
+ * Purpose: Create the collection of paths that will be searched
+ * when loading plugins.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__create_path_table(void)
+{
+ char *env_var= NULL; /* Path string from environment variable */
+ char *paths = NULL; /* Delimited paths string. Either from the
+ * environment variable or the default.
+ */
+ char *next_path = NULL; /* A path tokenized from the paths string */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Allocate memory for the path table */
+ H5PL_num_paths_g = 0;
+ H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
+ if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table")
+
+ /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it
+ * or from the default paths if it isn't set.
+ */
+ env_var = HDgetenv("HDF5_PLUGIN_PATH");
+ if (NULL == env_var)
+ paths = H5MM_strdup(H5PL_DEFAULT_PATH);
+ else
+ paths = H5MM_strdup(env_var);
+
+ if (NULL == paths)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy")
+
+ /* Separate the paths and store them */
+ /* XXX: strtok() is not thread-safe */
+ next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR);
+ while (next_path) {
+
+ /* Insert the path into the table */
+ if (H5PL__append_path(next_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path)
+
+ /* Get the next path from the environment string */
+ next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR);
+ } /* end while */
+
+done:
+ if (paths)
+ paths = (char *)H5MM_xfree(paths);
+
+ /* Try to clean up on errors */
+ if (FAIL == ret_value) {
+ if (H5PL_paths_g)
+ H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
+ H5PL_path_capacity_g = 0;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__create_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close_path_table
+ *
+ * Purpose: Close the collection of paths that will be searched
+ * when loading plugins.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close_path_table(void)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Free paths */
+ for (u = 0; u < H5PL_num_paths_g; u++)
+ if (H5PL_paths_g[u])
+ H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]);
+
+ /* Free path table */
+ H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
+
+ /* Reset values */
+ H5PL_num_paths_g = 0;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__close_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_num_paths
+ *
+ * Purpose: Gets the number of plugin paths that have been stored.
+ *
+ * Return: Success: The number of paths
+ * Failture: Can't fail
+ *-------------------------------------------------------------------------
+ */
+unsigned
+H5PL__get_num_paths(void)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ FUNC_LEAVE_NOAPI(H5PL_num_paths_g)
+
+} /* end H5PL__get_num_paths() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__expand_path_table
+ *
+ * Purpose: Expand the path table when it's full.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__expand_path_table(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Update the capacity */
+ H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD;
+
+ /* Resize the array */
+ if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed")
+
+ /* Initialize the new memory */
+ HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *));
+
+done:
+ /* Set the path capacity back if there were problems */
+ if (FAIL == ret_value)
+ H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__expand_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__append_path
+ *
+ * Purpose: Insert a path at the end of the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__append_path(const char *path)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Insert the path at the end of the table */
+ if (H5PL__insert_at(path, H5PL_num_paths_g) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__append_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__prepend_path
+ *
+ * Purpose: Insert a path at the beginning of the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__prepend_path(const char *path)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Insert the path at the beginning of the table */
+ if (H5PL__insert_at(path, 0) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__prepend_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__replace_path
+ *
+ * Purpose: Replace a path at particular index in the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__replace_path(const char *path, unsigned int index)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Insert the path at the requested index */
+ if (H5PL__replace_at(path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__insert_path
+ *
+ * Purpose: Insert a path at particular index in the table, moving
+ * any existing paths back to make space.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__insert_path(const char *path, unsigned int index)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Insert the path at the requested index */
+ if (H5PL__insert_at(path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__insert_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__remove_path
+ *
+ * Purpose: Remove a path at particular index in the table, freeing
+ * the path string and moving the paths down to close the gap.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__remove_path(unsigned int index)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Check if the path at that index is set */
+ if (!H5PL_paths_g[index])
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index)
+
+ /* Delete the path */
+ H5PL_num_paths_g--;
+ H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
+
+ /* Shift the paths down to close the gap */
+ for (u = index; u < H5PL_num_paths_g; u++)
+ H5PL_paths_g[u] = H5PL_paths_g[u+1];
+
+ /* Set the (former) last path to NULL */
+ H5PL_paths_g[H5PL_num_paths_g] = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__remove_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_path
+ *
+ * Purpose: Get a pointer to a path at particular index in the table.
+ *
+ * Return: Success: A pointer to a path string stored in the table
+ * Failure: NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+const char *
+H5PL__get_path(unsigned int index)
+{
+ char *ret_value = NULL; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Get the path at the requested index */
+ if (index >= H5PL_num_paths_g)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index)
+
+ return H5PL_paths_g[index];
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_path_table
+ *
+ * Purpose: Attempts to find a matching plugin in the file system
+ * using the paths stored in the path table.
+ *.
+ * The 'found' parameter will be set appropriately.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(plugin_info);
+
+ /* Initialize output parameters */
+ *found = FALSE;
+ *plugin_info = NULL;
+
+ /* Loop over the paths in the table, checking for an appropriate plugin */
+ for (u = 0; u < H5PL_num_paths_g; u++) {
+
+ /* Search for the plugin in this path */
+ if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u])
+
+ /* Break out if found */
+ if (*found) {
+ if (!plugin_info)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL")
+ break;
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_path
+ *
+ * Purpose: Given a path, this function opens the directory and envokes
+ * another function to go through all files to find the right
+ * plugin library. Two function definitions are for Unix and
+ * Windows.
+ *
+ * The found parameter will be set to TRUE and the info
+ * parameter will be filled in on success.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef H5_HAVE_WIN32_API
+static herr_t
+H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
+{
+ char *path = NULL;
+ DIR *dirp = NULL; /* Directory stream */
+ struct dirent *dp = NULL; /* Directory entry */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(dir);
+ HDassert(plugin_info);
+
+ /* Initialize the found parameter */
+ *found = FALSE;
+
+ /* Open the directory */
+ if (!(dirp = HDopendir(dir)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir)
+
+ /* Iterate through all entries in the directory */
+ while (NULL != (dp = HDreaddir(dirp))) {
+
+ /* The library we are looking for should be called libxxx.so... on Unix
+ * or libxxx.xxx.dylib on Mac.
+ */
+#ifndef __CYGWIN__
+ if (!HDstrncmp(dp->d_name, "lib", (size_t)3) &&
+ (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) {
+#else
+ if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) &&
+ HDstrstr(dp->d_name, ".dll") ) {
+#endif
+
+ h5_stat_t my_stat;
+ size_t len;
+
+ /* Allocate & initialize the path name */
+ len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/;
+
+ if (NULL == (path = (char *)H5MM_calloc(len)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+
+ HDsnprintf(path, len, "%s/%s", dir, dp->d_name);
+
+ /* Get info for directory entry */
+ if (HDstat(path, &my_stat) == -1)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno))
+
+ /* If it is a directory, skip it */
+ if (S_ISDIR(my_stat.st_mode))
+ continue;
+
+ /* attempt to open the dynamic library as a filter library */
+ if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
+ if (*found)
+ HGOTO_DONE(SUCCEED)
+
+ path = (char *)H5MM_xfree(path);
+ } /* end if */
+ } /* end while */
+
+done:
+ if (dirp)
+ if (HDclosedir(dirp) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno))
+
+ path = (char *)H5MM_xfree(path);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path() */
+#else /* H5_HAVE_WIN32_API */
+static herr_t
+H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
+{
+ WIN32_FIND_DATAA fdFile;
+ HANDLE hFind = INVALID_HANDLE_VALUE;
+ char *path = NULL;
+ char service[2048];
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(dir);
+ HDassert(plugin_info);
+
+ /* Initialize the found parameter */
+ *found = FALSE;
+
+ /* Specify a file mask. *.* = We want everything! */
+ HDsprintf(service, "%s\\*.dll", dir);
+ if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory")
+
+ /* Loop over all the files */
+ do {
+ /* Ignore '.' and '..' */
+ if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) {
+
+ /* XXX: Probably just continue here and move the code below over one tab */
+
+ size_t len;
+
+ /* Allocate & initialize the path name */
+ len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1;
+
+ if (NULL == (path = (char *)H5MM_calloc(len)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+
+ HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName);
+
+ /* Ignore directories */
+ if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ continue;
+
+ /* attempt to open the dynamic library as a filter library */
+ if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
+ if (*found)
+ HGOTO_DONE(SUCCEED)
+
+ path = (char *)H5MM_xfree(path);
+ }
+ } while (FindNextFileA(hFind, &fdFile));
+
+done:
+ if (hFind != INVALID_HANDLE_VALUE)
+ FindClose(hFind);
+ if (path)
+ path = (char *)H5MM_xfree(path);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path() */
+#endif /* H5_HAVE_WIN32_API */
+
diff --git a/src/H5PLpkg.h b/src/H5PLpkg.h
index e356893..0d1c271 100644
--- a/src/H5PLpkg.h
+++ b/src/H5PLpkg.h
@@ -11,6 +11,12 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*
+ * Purpose: This file contains declarations which are visible only within
+ * the H5PL package. Source files outside the H5PL package should
+ * include H5PLprivate.h instead.
+ */
+
#if !(defined H5PL_FRIEND || defined H5PL_MODULE)
#error "Do not include this file outside the H5PL package!"
#endif
@@ -27,13 +33,92 @@
/* Package Private Macros */
/**************************/
-#define H5PL_MAX_PATH_NUM 16
+/* Whether to pre-load pathnames for plugin libraries */
+#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR
+
+
+/****************************/
+/* Macros for supporting */
+/* both Windows and POSIX */
+/****************************/
+
+/*******************/
+/* Windows support */
+/*******************/
+/*
+ * SPECIAL WINDOWS NOTE
+ *
+ * Some of the Win32 API functions expand to fooA or fooW depending on
+ * whether UNICODE or _UNICODE are defined. You MUST explicitly use
+ * the A version of the functions to force char * behavior until we
+ * work out a scheme for proper Windows Unicode support.
+ *
+ * If you do not do this, people will be unable to incorporate our
+ * source code into their own CMake builds if they define UNICODE.
+ */
+#ifdef H5_HAVE_WIN32_API
+
+ /* The path separator on this platform */
+# define H5PL_PATH_SEPARATOR ";"
+
+ /* Handle for dynamic library */
+# define H5PL_HANDLE HINSTANCE
+
+ /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
+# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)
+
+ /* Get the address of a symbol in dynamic library */
+# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N)
+
+ /* Close dynamic library */
+# define H5PL_CLOSE_LIB(H) FreeLibrary(H)
+
+ /* Clear error - nothing to do */
+# define H5PL_CLR_ERROR
+
+ /* maximum size for expanding env vars */
+# define H5PL_EXPAND_BUFFER_SIZE 32767
+
+ typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void);
+
+#else /* H5_HAVE_WIN32_API */
+
+ /*****************/
+ /* POSIX support */
+ /*****************/
+
+ /* The path separator on this platform */
+# define H5PL_PATH_SEPARATOR ":"
+
+ /* Handle for dynamic library */
+# define H5PL_HANDLE void *
+
+ /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
+# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY)
+
+ /* Get the address of a symbol in dynamic library */
+# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N)
+
+ /* Close dynamic library */
+# define H5PL_CLOSE_LIB(H) dlclose(H)
+
+ /* Clear error */
+# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror())
+
+ typedef const void *(*H5PL_get_plugin_info_t)(void);
+#endif /* H5_HAVE_WIN32_API */
/****************************/
/* Package Private Typedefs */
/****************************/
+/* Data used to search for plugins */
+typedef struct H5PL_search_params_t {
+ H5PL_type_t type;
+ int id;
+} H5PL_search_params_t;
+
/*****************************/
/* Package Private Variables */
@@ -44,5 +129,31 @@
/* Package Private Prototypes */
/******************************/
+/* Accessors to global variables and flags */
+H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/);
+H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask);
+
+/* Plugin search and manipulation */
+H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/);
+H5_DLL herr_t H5PL__close(H5PL_HANDLE handle);
+
+/* Plugin cache calls */
+H5_DLL herr_t H5PL__create_plugin_cache(void);
+H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/);
+H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle);
+H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
+
+/* Plugin search path calls */
+H5_DLL herr_t H5PL__create_path_table(void);
+H5_DLL herr_t H5PL__close_path_table(void);
+H5_DLL unsigned H5PL__get_num_paths(void);
+H5_DLL herr_t H5PL__append_path(const char *path);
+H5_DLL herr_t H5PL__prepend_path(const char *path);
+H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index);
+H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index);
+H5_DLL herr_t H5PL__remove_path(unsigned int index);
+H5_DLL const char *H5PL__get_path(unsigned int index);
+H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
+
#endif /* _H5PLpkg_H */
diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c
new file mode 100644
index 0000000..d826ba0
--- /dev/null
+++ b/src/H5PLplugin_cache.c
@@ -0,0 +1,307 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Code to implement a plugin cache which stores information
+ * about plugins which have already been loaded.
+ *
+ * The plugin cache is implemented as a dynamic, global array which
+ * will grow as new plugins are added. The capacity of the cache
+ * never shrinks since plugins stay in memory once loaded.
+ *
+ * Note that this functionality has absolutely nothing to do with
+ * the metadata or chunk caches.
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+#include "H5Zprivate.h" /* Filter pipeline */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+/* Initial capacity of the plugin cache */
+#define H5PL_INITIAL_CACHE_CAPACITY 16
+
+/* The amount to add to the capacity when the cache is full */
+#define H5PL_CACHE_CAPACITY_ADD 16
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/* Type for the list of info for opened plugin libraries */
+typedef struct H5PL_plugin_t {
+ H5PL_type_t type; /* Plugin type */
+ int id; /* ID for the plugin */
+ H5PL_HANDLE handle; /* Plugin handle */
+} H5PL_plugin_t;
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static herr_t H5PL__expand_cache(void);
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Cache for storing opened plugin libraries */
+static H5PL_plugin_t *H5PL_cache_g = NULL;
+
+/* The number of stored plugins */
+static unsigned int H5PL_num_plugins_g = 0;
+
+/* The capacity of the plugin cache */
+static unsigned int H5PL_cache_capacity_g = 0;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__create_plugin_cache
+ *
+ * Purpose: Create the cache that will store plugins that have already
+ * been loaded.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__create_plugin_cache(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Allocate memory for the plugin cache */
+ H5PL_num_plugins_g = 0;
+
+ H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY;
+
+ if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache")
+
+done:
+ /* Try to clean up on errors */
+ if (FAIL == ret_value) {
+ if (H5PL_cache_g)
+ H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
+ H5PL_cache_capacity_g = 0;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__create_plugin_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close_plugin_cache
+ *
+ * Purpose: Close the cache of plugins that have already been loaded,
+ * closing all the plugins contained inside.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close_plugin_cache(hbool_t *already_closed /*out*/)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Close opened dynamic libraries */
+ if (H5PL_cache_g) {
+
+ /* Close any cached plugins */
+ for (u = 0; u < H5PL_num_plugins_g; u++)
+ H5PL__close((H5PL_cache_g[u]).handle);
+
+ /* Free the cache array */
+ H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
+ H5PL_num_plugins_g = 0;
+ H5PL_cache_capacity_g = 0;
+
+ /* Note that actually closed the table (needed by package close call) */
+ *already_closed = FALSE;
+ }
+ else
+ *already_closed = TRUE;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__close_plugin_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__expand_cache
+ *
+ * Purpose: Expand the plugin cache when it's full.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__expand_cache(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Update the capacity */
+ H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD;
+
+ /* Resize the array */
+ if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed")
+
+ /* Initialize the new memory */
+ HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t));
+
+done:
+ /* Set the cache capacity back if there were problems */
+ if (FAIL == ret_value)
+ H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__expand_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__add_plugin
+ *
+ * Purpose: Add a plugin to the plugin cached.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Expand the cache if it is too small */
+ if (H5PL_num_plugins_g >= H5PL_cache_capacity_g)
+ if (H5PL__expand_cache() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache")
+
+ /* Store the plugin info and bump the # of plugins */
+ H5PL_cache_g[H5PL_num_plugins_g].type = type;
+ H5PL_cache_g[H5PL_num_plugins_g].id = id;
+ H5PL_cache_g[H5PL_num_plugins_g].handle = handle;
+
+ H5PL_num_plugins_g++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__add_plugin() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_cache
+ *
+ * Purpose: Attempts to find a matching plugin from the cache.
+ *
+ * The 'found' parameter will be set appropriately.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+/* See the other use of H5PL_GET_LIB_FUNC() for an explanation
+ * for why we disable -Wpedantic here.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+herr_t
+H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(plugin_info);
+
+ /* Initialize output parameters */
+ *found = FALSE;
+ *plugin_info = NULL;
+
+ /* Loop over all the plugins, looking for one that matches */
+ for (u = 0; u < H5PL_num_plugins_g; u++) {
+
+ /* If the plugin type (filter, etc.) and ID match, query the plugin for its info */
+ if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) {
+
+ H5PL_get_plugin_info_t get_plugin_info_function;
+ const H5Z_class2_t *filter_info;
+
+ /* Get the "get plugin info" function from the plugin. */
+ if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info")))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info")
+
+ /* Call the "get plugin info" function */
+ if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)()))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info")
+
+ /* Set output parameters */
+ *found = TRUE;
+ *plugin_info = filter_info;
+
+ /* No need to continue processing */
+ break;
+
+ } /* end if */
+
+ } /* end for */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_cache() */
+#pragma GCC diagnostic pop
+
diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h
index 9ce1fca..3b36ccd 100644
--- a/src/H5PLpublic.h
+++ b/src/H5PLpublic.h
@@ -10,8 +10,8 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* Programmer: Raymond Lu <songyulu@hdfgroup.org>
- * 13 February 2013
+/*
+ * This file contains public declarations for the H5PL module.
*/
#ifndef _H5PLpublic_H
@@ -24,31 +24,35 @@
/* Public Typedefs */
/*******************/
+/* Special string to indicate no plugin loading.
+ */
+#define H5PL_NO_PLUGIN "::"
+
/* Plugin type used by the plugin library */
typedef enum H5PL_type_t {
- H5PL_TYPE_ERROR = -1, /*error */
- H5PL_TYPE_FILTER = 0, /*filter */
- H5PL_TYPE_NONE = 1 /*this must be last! */
+ H5PL_TYPE_ERROR = -1, /* Error */
+ H5PL_TYPE_FILTER = 0, /* Filter */
+ H5PL_TYPE_NONE = 1 /* This must be last! */
} H5PL_type_t;
/* Common dynamic plugin type flags used by the set/get_loading_state functions */
-#define H5PL_FILTER_PLUGIN 0x0001
-#define H5PL_ALL_PLUGIN 0xFFFF
+#define H5PL_FILTER_PLUGIN 0x0001
+#define H5PL_ALL_PLUGIN 0xFFFF
#ifdef __cplusplus
extern "C" {
#endif
/* plugin state */
-H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type);
-H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/);
-H5_DLL herr_t H5PLappend(const char *plugin_path);
-H5_DLL herr_t H5PLprepend(const char *plugin_path);
-H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index);
-H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index);
+H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask);
+H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/);
+H5_DLL herr_t H5PLappend(const char *search_path);
+H5_DLL herr_t H5PLprepend(const char *search_path);
+H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index);
+H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index);
H5_DLL herr_t H5PLremove(unsigned int index);
-H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size);
-H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/);
+H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size);
+H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/);
#ifdef __cplusplus
}
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 55b3877..854b1ef 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -166,7 +166,7 @@ typedef enum H5D_mpio_no_collective_cause_t {
H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08,
H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10,
H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x20,
- H5D_MPIO_FILTERS = 0x40
+ H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x40
} H5D_mpio_no_collective_cause_t;
/********************/
diff --git a/src/H5R.c b/src/H5R.c
index 73c1d55..b000183 100644
--- a/src/H5R.c
+++ b/src/H5R.c
@@ -105,7 +105,7 @@ H5R__init_package(void)
/* Initialize the atom group for the file IDs */
if(H5I_register_type(H5I_REFERENCE_CLS) < 0)
- HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface")
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface")
/* Mark "top" of interface as initialized, too */
H5R_top_package_initialize_s = TRUE;
@@ -141,10 +141,10 @@ H5R_top_term_package(void)
FUNC_ENTER_NOAPI_NOINIT_NOERR
if(H5R_top_package_initialize_s) {
- if(H5I_nmembers(H5I_REFERENCE) > 0) {
- (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE);
+ if(H5I_nmembers(H5I_REFERENCE) > 0) {
+ (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE);
n++; /*H5I*/
- } /* end if */
+ } /* end if */
/* Mark closed */
if(0 == n)
@@ -741,7 +741,7 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTCREATE, FAIL, "unable to create dataspace")
/* Atomize */
- if((ret_value = H5I_register (H5I_DATASPACE, space, TRUE)) < 0)
+ if((ret_value = H5I_register(H5I_DATASPACE, space, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace atom")
done:
diff --git a/src/H5err.txt b/src/H5err.txt
index 3f5801f..d771956 100644
--- a/src/H5err.txt
+++ b/src/H5err.txt
@@ -243,6 +243,8 @@ MINOR, LINK, H5E_CANTSORT, Can't sort objects
MINOR, MPI, H5E_MPI, Some MPI function failed
MINOR, MPI, H5E_MPIERRSTR, MPI Error String
MINOR, MPI, H5E_CANTRECV, Can't receive data
+MINOR, MPI, H5E_CANTGATHER, Can't gather data
+MINOR, MPI, H5E_NO_INDEPENDENT, Can't perform independent IO
# Heap errors
MINOR, HEAP, H5E_CANTRESTORE, Can't restore condition
diff --git a/src/H5private.h b/src/H5private.h
index c588154..ca4ebcf 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -495,6 +495,15 @@
# define H5_POSIX_MAX_IO_BYTES SSIZET_MAX
#endif
+/* POSIX I/O mode used as the third parameter to open/_open
+ * when creating a new file (O_CREAT is set).
+ */
+#if defined(H5_HAVE_WIN32_API)
+# define H5_POSIX_CREATE_MODE_RW (_S_IREAD | _S_IWRITE)
+#else
+# define H5_POSIX_CREATE_MODE_RW 0666
+#endif
+
/*
* A macro to portably increment enumerated types.
*/
@@ -1117,11 +1126,7 @@ typedef off_t h5_stat_size_t;
#define HDnanosleep(N, O) nanosleep(N, O)
#endif /* HDnanosleep */
#ifndef HDopen
- #ifdef _O_BINARY
- #define HDopen(S,F,M) open(S,F|_O_BINARY,M)
- #else
- #define HDopen(S,F,M) open(S,F,M)
- #endif
+ #define HDopen(F,...) open(F,__VA_ARGS__)
#endif /* HDopen */
#ifndef HDopendir
#define HDopendir(S) opendir(S)
diff --git a/src/H5system.c b/src/H5system.c
index 7e25540..a1cdf19 100644
--- a/src/H5system.c
+++ b/src/H5system.c
@@ -1237,3 +1237,49 @@ H5_get_time(void)
} /* end H5_get_time() */
+#ifdef H5_HAVE_WIN32_API
+
+#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5_expand_windows_env_vars()
+ *
+ * Purpose: Replaces windows environment variables of the form %foo%
+ * with user-specific values.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5_expand_windows_env_vars(char **env_var)
+{
+ long n_chars = 0;
+ char *temp_buf = NULL;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Allocate buffer for expanded environment variable string */
+ if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path")
+
+ /* Expand the environment variable string */
+ if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long")
+
+ if (0 == n_chars)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path")
+
+ *env_var = (char *)H5MM_xfree(*env_var);
+ *env_var = temp_buf;
+
+done:
+ if (FAIL == ret_value && temp_buf)
+ temp_buf = (char *)H5MM_xfree(temp_buf);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5_expand_windows_env_vars() */
+#endif /* H5_HAVE_WIN32_API */
+
diff --git a/src/H5trace.c b/src/H5trace.c
index 9fb8a72..930002f 100644
--- a/src/H5trace.c
+++ b/src/H5trace.c
@@ -621,10 +621,6 @@ H5_trace(const double *returning, const char *func, const char *type, ...)
fprintf(out, "%sH5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET", flag_already_displayed ? " | " : "");
flag_already_displayed = TRUE;
} /* end if */
- if(nocol_cause_mode & H5D_MPIO_FILTERS) {
- fprintf(out, "%sH5D_MPIO_FILTERS", flag_already_displayed ? " | " : "");
- flag_already_displayed = TRUE;
- } /* end if */
/* Display '<none>' if there's no flags set */
if(!flag_already_displayed)
diff --git a/src/H5win32defs.h b/src/H5win32defs.h
index 0149faa..4522228 100644
--- a/src/H5win32defs.h
+++ b/src/H5win32defs.h
@@ -48,9 +48,13 @@ typedef __int64 h5_stat_size_t;
#define HDnanosleep(N, O) Wnanosleep(N, O)
#define HDoff_t __int64
/* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL
- * transformations when performing I/O.
+ * transformations when performing I/O. Note that this will
+ * produce Unix-style text files, though.
+ *
+ * Also note that the variadic macro is using a VC++ extension
+ * where the comma is dropped if nothing is passed to the ellipsis.
*/
-#define HDopen(S,F,M) _open(S,F|_O_BINARY,M)
+#define HDopen(S,F,...) _open(S, F | _O_BINARY, __VA_ARGS__)
#define HDread(F,M,Z) _read(F,M,Z)
#define HDrmdir(S) _rmdir(S)
#define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,(Z>1?Z:2))
@@ -115,6 +119,7 @@ extern "C" {
H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...);
H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap);
H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem);
+ H5_DLL herr_t H5_expand_windows_env_vars(char **env_var);
/* Round functions only needed for VS2012 and earlier.
* They are always built to ensure they don't go stale and
diff --git a/src/Makefile.am b/src/Makefile.am
index 0b664a7..9a64717 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Pgcpl.c H5Pint.c \
H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \
H5PB.c \
- H5PL.c \
+ H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \
H5R.c H5Rdeprec.c \
H5UC.c \
H5RS.c \