summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorM. Scot Breitenfeld <brtnfld@hdfgroup.org>2017-10-26 15:13:33 (GMT)
committerM. Scot Breitenfeld <brtnfld@hdfgroup.org>2017-10-26 15:13:33 (GMT)
commit59e7c7703e62717e7bf482bec2bbad52aada3c80 (patch)
tree59ebe294f77f57850a017ff9596234066d3acebd /src
parentc3aa3978b2873b158a1917f8961c035f413b5e56 (diff)
parent084704392fa93d1569861c53504e89236468a3b6 (diff)
downloadhdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.zip
hdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.tar.gz
hdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.tar.bz2
Merge branch 'develop' into hdf5_1_10
Diffstat (limited to 'src')
-rw-r--r--src/CMakeLists.txt3
-rw-r--r--src/H5.c7
-rw-r--r--src/H5ACprivate.h3
-rw-r--r--src/H5C.c81
-rw-r--r--src/H5Cdbg.c2
-rw-r--r--src/H5Cimage.c2
-rw-r--r--src/H5Cpkg.h2
-rw-r--r--src/H5Cprivate.h2
-rw-r--r--src/H5Dchunk.c5
-rw-r--r--src/H5Defl.c4
-rw-r--r--src/H5Dint.c4
-rw-r--r--src/H5Dio.c68
-rw-r--r--src/H5Dmpio.c1665
-rw-r--r--src/H5Dpkg.h8
-rw-r--r--src/H5Dscatgath.c7
-rw-r--r--src/H5F.c22
-rw-r--r--src/H5FDcore.c10
-rw-r--r--src/H5FDdirect.c2
-rw-r--r--src/H5FDlog.c2
-rw-r--r--src/H5FDmpi.c5
-rw-r--r--src/H5FDsec2.c2
-rw-r--r--src/H5FSint.c26
-rw-r--r--src/H5FSprivate.h3
-rw-r--r--src/H5Fint.c17
-rw-r--r--src/H5Fpkg.h30
-rw-r--r--src/H5Fsuper.c136
-rw-r--r--src/H5I.c4
-rw-r--r--src/H5Ipublic.h9
-rw-r--r--src/H5MF.c1
-rw-r--r--src/H5MFdbg.c2
-rw-r--r--src/H5Oefl.c2
-rw-r--r--src/H5Oflush.c38
-rw-r--r--src/H5Otest.c2
-rw-r--r--src/H5PB.c4
-rw-r--r--src/H5PL.c984
-rw-r--r--src/H5PLint.c384
-rw-r--r--src/H5PLpath.c776
-rw-r--r--src/H5PLpkg.h113
-rw-r--r--src/H5PLplugin_cache.c307
-rw-r--r--src/H5PLpublic.h34
-rw-r--r--src/H5Ppublic.h2
-rw-r--r--src/H5R.c10
-rw-r--r--src/H5err.txt2
-rw-r--r--src/H5private.h15
-rw-r--r--src/H5system.c46
-rw-r--r--src/H5trace.c4
-rw-r--r--src/H5win32defs.h9
-rw-r--r--src/Makefile.am2
48 files changed, 3829 insertions, 1039 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 178c954..96ea589 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" )
set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
+ ${HDF5_SRC_DIR}/H5PLint.c
+ ${HDF5_SRC_DIR}/H5PLpath.c
+ ${HDF5_SRC_DIR}/H5PLplugin_cache.c
)
set (H5PL_HDRS
diff --git a/src/H5.c b/src/H5.c
index 1068fc6..59984dc 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -29,6 +29,7 @@
#include "H5Pprivate.h" /* Property lists */
#include "H5SLprivate.h" /* Skip lists */
#include "H5Tprivate.h" /* Datatypes */
+#include "H5FSprivate.h" /* File free space */
/****************/
/* Local Macros */
@@ -204,6 +205,10 @@ H5_init_library(void)
* property classes.
* The link interface needs to be initialized so that link property lists
* have their properties registered.
+ * The FS module needs to be initialized as a result of the fix for HDFFV-10160:
+ * It might not be initialized during normal file open.
+ * When the application does not close the file, routines in the module might
+ * be called via H5_term_library() when shutting down the file.
*/
if(H5E_init() < 0)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface")
@@ -217,6 +222,8 @@ H5_init_library(void)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize metadata caching interface")
if(H5L_init() < 0)
HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize link interface")
+ if(H5FS_init() < 0)
+ HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize FS interface")
/* Debugging? */
H5_debug_mask("-all");
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index b9e2a60..083ee5b 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -230,7 +230,7 @@ typedef struct H5AC_proxy_entry_t {
/* (Note that this currently duplicates some cache functionality) */
} H5AC_proxy_entry_t;
-
+/* Name of property for ring info in DXPL */
#define H5AC_RING_NAME "H5AC_ring_type"
/* Dataset transfer property lists for metadata calls */
@@ -247,7 +247,6 @@ H5_DLLVAR hid_t H5AC_noio_dxpl_id;
H5_DLLVAR hid_t H5AC_rawdata_dxpl_id;
/* Default cache configuration. */
-
#define H5AC__DEFAULT_METADATA_WRITE_STRATEGY \
H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED
diff --git a/src/H5C.c b/src/H5C.c
index e6770ec..0d8cc75 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -366,6 +366,7 @@ H5C_create(size_t max_cache_size,
cache_ptr->coll_write_list = NULL;
#endif /* H5_HAVE_PARALLEL */
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
cache_ptr->cLRU_list_len = 0;
cache_ptr->cLRU_list_size = (size_t)0;
cache_ptr->cLRU_head_ptr = NULL;
@@ -375,6 +376,7 @@ H5C_create(size_t max_cache_size,
cache_ptr->dLRU_list_size = (size_t)0;
cache_ptr->dLRU_head_ptr = NULL;
cache_ptr->dLRU_tail_ptr = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
cache_ptr->size_increase_possible = FALSE;
cache_ptr->flash_size_increase_possible = FALSE;
@@ -1481,8 +1483,10 @@ H5C_insert_entry(H5F_t * f,
entry_ptr->next = NULL;
entry_ptr->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
entry_ptr->aux_next = NULL;
entry_ptr->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
entry_ptr->coll_next = NULL;
@@ -4062,8 +4066,7 @@ H5C_destroy_flush_dependency(void *parent_thing, void * child_thing)
child_entry->flush_dep_parent_nalloc = 0;
} /* end if */
else if(child_entry->flush_dep_parent_nalloc > H5C_FLUSH_DEP_PARENT_INIT
- && child_entry->flush_dep_nparents
- <= (child_entry->flush_dep_parent_nalloc / 4)) {
+ && child_entry->flush_dep_nparents <= (child_entry->flush_dep_parent_nalloc / 4)) {
if(NULL == (child_entry->flush_dep_parent = (H5C_cache_entry_t **)H5FL_BLK_REALLOC(parent, child_entry->flush_dep_parent, (child_entry->flush_dep_parent_nalloc / 4) * sizeof(H5C_cache_entry_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for flush dependency parent list")
child_entry->flush_dep_parent_nalloc /= 4;
@@ -6273,17 +6276,27 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item")
} /* end if */
else
+ {
#endif /* H5_HAVE_PARALLEL */
- if(entry_ptr->prefetched) {
- HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID);
- mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type;
- } /* end if */
- else
- mem_type = entry_ptr->type->mem_type;
+ if(entry_ptr->prefetched) {
+ HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID);
+ mem_type = cache_ptr->
+ class_table_ptr[entry_ptr->prefetch_type_id]->
+ mem_type;
+ } /* end if */
+ else
+ mem_type = entry_ptr->type->mem_type;
+
+ if(H5F_block_write(f, mem_type, entry_ptr->addr,
+ entry_ptr->size, dxpl_id,
+ entry_ptr->image_ptr) < 0)
- if(H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+ "Can't write image to file")
+#ifdef H5_HAVE_PARALLEL
+ }
+#endif /* H5_HAVE_PARALLEL */
} /* end if */
/* if the entry has a notify callback, notify it that we have
@@ -6418,7 +6431,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
HDassert(entry_ptr->flush_dep_ndirty_children == 0);
if(entry_ptr->flush_dep_nparents > 0)
if(H5C__mark_flush_dep_clean(entry_ptr) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean flag")
} /* end if */
} /* end else */
@@ -6940,8 +6953,10 @@ H5C_load_entry(H5F_t * f,
entry->next = NULL;
entry->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
entry->aux_next = NULL;
entry->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
entry->coll_next = NULL;
@@ -7276,6 +7291,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
HDassert( H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS );
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
initial_list_len = cache_ptr->cLRU_list_len;
entry_ptr = cache_ptr->cLRU_tail_ptr;
@@ -7320,6 +7336,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
entry_ptr = prev_ptr;
entries_examined++;
}
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
}
done:
@@ -7979,7 +7996,7 @@ done:
static herr_t
H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry)
{
- unsigned u; /* Local index variable */
+ int i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -7988,16 +8005,19 @@ H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry)
HDassert(entry);
/* Iterate over the parent entries, if any */
- for(u = 0; u < entry->flush_dep_nparents; u++) {
+ /* Note reverse iteration order, in case the callback removes the flush
+ * dependency - QAK, 2017/08/12
+ */
+ for(i = ((int)entry->flush_dep_nparents) - 1; i >= 0; i--) {
/* Sanity check */
- HDassert(entry->flush_dep_parent[u]->flush_dep_ndirty_children > 0);
+ HDassert(entry->flush_dep_parent[i]->flush_dep_ndirty_children > 0);
/* Adjust the parent's number of dirty children */
- entry->flush_dep_parent[u]->flush_dep_ndirty_children--;
+ entry->flush_dep_parent[i]->flush_dep_ndirty_children--;
/* If the parent has a 'notify' callback, send a 'child entry cleaned' notice */
- if(entry->flush_dep_parent[u]->type->notify &&
- (entry->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[u]) < 0)
+ if(entry->flush_dep_parent[i]->type->notify &&
+ (entry->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[i]) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag reset")
} /* end for */
@@ -8023,7 +8043,7 @@ done:
herr_t
H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr)
{
- unsigned u; /* Local index variable */
+ int i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -8032,18 +8052,21 @@ H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr)
HDassert(entry_ptr);
/* Iterate over the parent entries, if any */
- for(u = 0; u < entry_ptr->flush_dep_nparents; u++) {
-
+ /* Note reverse iteration order, in case the callback removes the flush
+ * dependency - QAK, 2017/08/12
+ */
+ for(i = ((int)entry_ptr->flush_dep_nparents) - 1; i >= 0; i--) {
+ /* Sanity checks */
HDassert(entry_ptr->flush_dep_parent);
- HDassert(entry_ptr->flush_dep_parent[u]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
- HDassert(entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children > 0);
+ HDassert(entry_ptr->flush_dep_parent[i]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+ HDassert(entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children > 0);
/* decrement the parents number of unserialized children */
- entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children--;
+ entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children--;
/* If the parent has a 'notify' callback, send a 'child entry serialized' notice */
- if(entry_ptr->flush_dep_parent[u]->type->notify &&
- (entry_ptr->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[u]) < 0)
+ if(entry_ptr->flush_dep_parent[i]->type->notify &&
+ (entry_ptr->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[i]) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry serialized flag set")
} /* end for */
@@ -8055,7 +8078,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5C__mark_flush_dep_unserialized()
*
- * Purpose: Decrement the flush_dep_nunser_children fields of all the
+ * Purpose: Increment the flush_dep_nunser_children fields of all the
* target entry's flush dependency parents in response to
* the target entry becoming unserialized.
*
@@ -8862,6 +8885,12 @@ H5C_remove_entry(void *_entry)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove protected entry from cache")
if(entry->is_pinned)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove pinned entry from cache")
+ /* NOTE: If these two errors are getting tripped because the entry is
+ * in a flush dependency with a freedspace entry, move the checks
+ * after the "before evict" message is sent, and add the
+ * "child being evicted" message to the "before evict" notify
+ * section below. QAK - 2017/08/03
+ */
if(entry->flush_dep_nparents > 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry with flush dependency parents from cache")
if(entry->flush_dep_nchildren > 0)
diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c
index 4a08d9b..08c70d9 100644
--- a/src/H5Cdbg.c
+++ b/src/H5Cdbg.c
@@ -716,6 +716,7 @@ H5C_stats(H5C_t * cache_ptr,
(long)(cache_ptr->LRU_list_size),
(unsigned long)(cache_ptr->LRU_list_len));
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
HDfprintf(stdout,
"%s current clean LRU size / length = %ld / %lu\n",
cache_ptr->prefix,
@@ -727,6 +728,7 @@ H5C_stats(H5C_t * cache_ptr,
cache_ptr->prefix,
(long)(cache_ptr->dLRU_list_size),
(unsigned long)(cache_ptr->dLRU_list_len));
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
HDfprintf(stdout,
"%s Total hits / misses / hit_rate = %ld / %ld / %f\n",
diff --git a/src/H5Cimage.c b/src/H5Cimage.c
index debd30c..53d1712 100644
--- a/src/H5Cimage.c
+++ b/src/H5Cimage.c
@@ -649,8 +649,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
/* Initialize fields supporting replacement policies: */
ds_entry_ptr->next = NULL;
ds_entry_ptr->prev = NULL;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
ds_entry_ptr->aux_next = NULL;
ds_entry_ptr->aux_prev = NULL;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
pf_entry_ptr->coll_next = NULL;
pf_entry_ptr->coll_prev = NULL;
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index fdb14a5..d431887 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -4746,6 +4746,7 @@ struct H5C_t {
H5C_cache_entry_t * LRU_head_ptr;
H5C_cache_entry_t * LRU_tail_ptr;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
/* Fields for clean LRU list of entries */
uint32_t cLRU_list_len;
size_t cLRU_list_size;
@@ -4757,6 +4758,7 @@ struct H5C_t {
size_t dLRU_list_size;
H5C_cache_entry_t * dLRU_head_ptr;
H5C_cache_entry_t * dLRU_tail_ptr;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
/* Fields for collective metadata reads */
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 5335f80..50732ca 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -1648,8 +1648,10 @@ typedef struct H5C_cache_entry_t {
/* fields supporting replacement policies: */
struct H5C_cache_entry_t *next;
struct H5C_cache_entry_t *prev;
+#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
struct H5C_cache_entry_t *aux_next;
struct H5C_cache_entry_t *aux_prev;
+#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
#ifdef H5_HAVE_PARALLEL
struct H5C_cache_entry_t *coll_next;
struct H5C_cache_entry_t *coll_prev;
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index b7b8b03..af6599a 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -299,9 +299,6 @@ static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info,
static herr_t H5D__chunk_cache_prune(const H5D_t *dset, hid_t dxpl_id,
const H5D_dxpl_cache_t *dxpl_cache, size_t size);
static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk);
-static herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info,
- const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert,
- hsize_t scaled[]);
#ifdef H5_HAVE_PARALLEL
static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id,
H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf);
@@ -6558,7 +6555,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-static herr_t
+herr_t
H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk,
H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[])
{
diff --git a/src/H5Defl.c b/src/H5Defl.c
index 5536ba3..ebe7689 100644
--- a/src/H5Defl.c
+++ b/src/H5Defl.c
@@ -288,7 +288,7 @@ H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size
HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed")
if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0)
HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name")
- if((fd = HDopen(full_name, O_RDONLY, 0)) < 0)
+ if((fd = HDopen(full_name, O_RDONLY)) < 0)
HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file")
if(HDlseek(fd, (HDoff_t)(efl->slot[u].offset + (HDoff_t)skip), SEEK_SET) < 0)
HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file")
@@ -380,7 +380,7 @@ H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t siz
HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed")
if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0)
HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name")
- if((fd = HDopen(full_name, O_CREAT | O_RDWR, 0666)) < 0) {
+ if((fd = HDopen(full_name, O_CREAT | O_RDWR, H5_POSIX_CREATE_MODE_RW)) < 0) {
if(HDaccess(full_name, F_OK) < 0)
HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist")
else
diff --git a/src/H5Dint.c b/src/H5Dint.c
index 3b938e2..bdedd1e 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -1213,10 +1213,6 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
/* Don't allow compact datasets to allocate space later */
if(layout->type == H5D_COMPACT && fill->alloc_time != H5D_ALLOC_TIME_EARLY)
HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "compact dataset must have early space allocation")
-
- /* If MPI VFD is used, no filter support yet. */
- if(H5F_HAS_FEATURE(file, H5FD_FEAT_HAS_MPI) && pline->nused > 0)
- HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, NULL, "Parallel I/O does not support filters yet")
} /* end if */
/* Set the latest version of the layout, pline & fill messages, if requested */
diff --git a/src/H5Dio.c b/src/H5Dio.c
index 1766422..104a632 100644
--- a/src/H5Dio.c
+++ b/src/H5Dio.c
@@ -714,11 +714,6 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
if(H5T_get_class(type_info.mem_type, TRUE) == H5T_REFERENCE &&
H5T_get_ref_type(type_info.mem_type) == H5R_DATASET_REGION)
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing region reference datatypes yet")
-
- /* Can't write to chunked datasets with filters, in parallel */
- if(dataset->shared->layout.type == H5D_CHUNKED &&
- dataset->shared->dcpl_cache.pline.nused > 0)
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot write to chunked storage with filters in parallel")
} /* end if */
else {
/* Collective access is not permissible without a MPI based VFD */
@@ -1195,7 +1190,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id,
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator")
/* Check if we can set direct MPI-IO read/write functions */
- if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, fm, dx_plist)) < 0)
+ if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, dx_plist)) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for direct IO dataspace ")
/* Check if we can use the optimized parallel I/O routines */
@@ -1207,6 +1202,67 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id,
io_info->io_ops.single_write = H5D__mpio_select_write;
} /* end if */
else {
+ /* Check if there are any filters in the pipeline. If there are,
+ * we cannot break to independent I/O if this is a write operation;
+ * otherwise there will be metadata inconsistencies in the file.
+ */
+ if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ H5D_mpio_no_collective_cause_t cause;
+ uint32_t local_no_collective_cause;
+ uint32_t global_no_collective_cause;
+ hbool_t local_error_message_previously_written = FALSE;
+ hbool_t global_error_message_previously_written = FALSE;
+ size_t index;
+ char local_no_collective_cause_string[256] = "";
+ char global_no_collective_cause_string[256] = "";
+ const char *cause_strings[] = { "independent I/O was requested",
+ "datatype conversions were required",
+ "data transforms needed to be applied",
+ "optimized MPI types flag wasn't set",
+ "one of the dataspaces was neither simple nor scalar",
+ "dataset was not contiguous or chunked" };
+
+ if (H5P_get(dx_plist, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, &local_no_collective_cause) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get local no collective cause value")
+ if (H5P_get(dx_plist, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, &global_no_collective_cause) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get global no collective cause value")
+
+ /* Append each of the "reason for breaking collective I/O" error messages to the
+ * local and global no collective cause strings */
+ for (cause = 1, index = 0; cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE; cause <<= 1, index++) {
+ size_t cause_strlen = strlen(cause_strings[index]);
+
+ if (cause & local_no_collective_cause) {
+ /* Check if there were any previous error messages included. If so, prepend a semicolon
+ * to separate the messages.
+ */
+ if (local_error_message_previously_written) strncat(local_no_collective_cause_string, "; ", 2);
+
+ strncat(local_no_collective_cause_string, cause_strings[index], cause_strlen);
+
+ local_error_message_previously_written = TRUE;
+ } /* end if */
+
+ if (cause & global_no_collective_cause) {
+ /* Check if there were any previous error messages included. If so, prepend a semicolon
+ * to separate the messages.
+ */
+ if (global_error_message_previously_written) strncat(global_no_collective_cause_string, "; ", 2);
+
+ strncat(global_no_collective_cause_string, cause_strings[index], cause_strlen);
+
+ global_error_message_previously_written = TRUE;
+ } /* end if */
+ } /* end for */
+
+ HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, "Can't perform independent write with filters in pipeline.\n"
+ " The following caused a break from collective I/O:\n"
+ " Local causes: %s\n"
+ " Global causes: %s",
+ local_no_collective_cause_string,
+ global_no_collective_cause_string);
+ } /* end if */
+
/* If we won't be doing collective I/O, but the user asked for
* collective I/O, change the request to use independent I/O, but
* mark it so that we remember to revert the change.
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 0389c72..79572c0 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -84,7 +84,6 @@
#define H5D_CHUNK_SELECT_IRREG 2
#define H5D_CHUNK_SELECT_NONE 0
-
/******************/
/* Local Typedefs */
/******************/
@@ -94,6 +93,113 @@ typedef struct H5D_chunk_addr_info_t {
H5D_chunk_info_t chunk_info;
} H5D_chunk_addr_info_t;
+/*
+ * Information about a single chunk when performing collective filtered I/O. All
+ * of the fields of one of these structs are initialized at the start of collective
+ * filtered I/O in the function H5D__construct_filtered_io_info_list().
+ *
+ * This struct's fields are as follows:
+ *
+ * index - The "Index" of the chunk in the dataset. The index of a chunk is used during
+ * the collective re-insertion of chunks into the chunk index after the collective
+ * I/O has been performed.
+ *
+ * scaled - The scaled coordinates of the chunk in the dataset's file dataspace. The
+ * coordinates are used in both the collective re-allocation of space in the file
+ * and the collective re-insertion of chunks into the chunk index after the collective
+ * I/O has been performed.
+ *
+ * full_overwrite - A flag which determines whether or not a chunk needs to be read from the
+ * file when being updated. If a chunk is being fully overwritten (the entire
+ * extent is selected in its file dataspace), then it is not necessary to
+ * read the chunk from the file. However, if the chunk is not being fully
+ * overwritten, it has to be read from the file in order to update the chunk
+ * without trashing the parts of the chunk that are not selected.
+ *
+ * num_writers - The total number of processors writing to this chunk. This field is used
+ * when the new owner of a chunk is receiving messages, which contain selections in
+ * the chunk and data to update the chunk with, from other processors which have this
+ * chunk selected in the I/O operation. The new owner must know how many processors it
+ * should expect messages from so that it can post an equal number of receive calls.
+ *
+ * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of
+ * I/O to the chunk from each processor which has the chunk selected and is used to
+ * determine the value for the previous full_overwrite flag.
+ *
+ * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be
+ * written to the file or the chunk data which has been read from the file.
+ *
+ * chunk_states - In the case of dataset writes only, this struct is used to track a chunk's size and
+ * address in the file before and after the filtering operation has occurred.
+ *
+ * Its fields are as follows:
+ *
+ * chunk_current - The address in the file and size of this chunk before the filtering
+ * operation. When reading a chunk from the file, this field is used to
+ * read the correct amount of bytes. It is also used when redistributing
+ * shared chunks among processors and as a parameter to the chunk file
+ * space reallocation function.
+ *
+ * new_chunk - The address in the file and size of this chunk after the filtering
+ * operation. This field is relevant when collectively re-allocating space
+ * in the file for all of the chunks written to in the I/O operation, as
+ * their sizes may have changed after their data has been filtered.
+ *
+ * owners - In the case of dataset writes only, this struct is used to manage which single processor
+ * will ultimately write data out to the chunk. It allows the other processors to act according
+ * to the decision and send their selection in the chunk, as well as the data they wish
+ * to update the chunk with, to the processor which is writing to the chunk.
+ *
+ * Its fields are as follows:
+ *
+ * original_owner - The processor which originally had this chunk selected at the beginning of
+ * the collective filtered I/O operation. This field is currently used when
+ * redistributing shared chunks among processors.
+ *
+ * new_owner - The processor which has been selected to perform the write to this chunk.
+ *
+ * async_info - In the case of dataset writes only, this struct is used by the owning processor of the
+ * chunk in order to manage the MPI send and receive calls made between it and all of
+ * the other processors which have this chunk selected in the I/O operation.
+ *
+ * Its fields are as follows:
+ *
+ * receive_requests_array - An array containing one MPI_Request for each of the
+ * asynchronous MPI receive calls the owning processor of this
+ * chunk makes to another processor in order to receive that
+ * processor's chunk modification data and selection in the chunk.
+ *
+ * receive_buffer_array - An array of buffers into which the owning processor of this chunk
+ * will store chunk modification data and the selection in the chunk
+ * received from another processor.
+ *
+ * num_receive_requests - The number of entries in the receive_request_array and
+ * receive_buffer_array fields.
+ */
+typedef struct H5D_filtered_collective_io_info_t {
+ hsize_t index;
+ hsize_t scaled[H5O_LAYOUT_NDIMS];
+ hbool_t full_overwrite;
+ size_t num_writers;
+ size_t io_size;
+ void *buf;
+
+ struct {
+ H5F_block_t chunk_current;
+ H5F_block_t new_chunk;
+ } chunk_states;
+
+ struct {
+ int original_owner;
+ int new_owner;
+ } owners;
+
+ struct {
+ MPI_Request *receive_requests_array;
+ unsigned char **receive_buffer_array;
+ int num_receive_requests;
+ } async_info;
+} H5D_filtered_collective_io_info_t;
/********************/
/* Local Prototypes */
@@ -103,9 +209,15 @@ static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info,
static herr_t H5D__multi_chunk_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
H5P_genplist_t *dx_plist);
+static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
+ H5P_genplist_t *dx_plist);
static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, int sum_chunk,
H5P_genplist_t *dx_plist);
+static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, H5D_chunk_map_t *fm,
+ H5P_genplist_t *dx_plist);
static herr_t H5D__inter_collective_io(H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, const H5S_t *file_space,
const H5S_t *mem_space);
@@ -124,6 +236,26 @@ static herr_t H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info,
const H5D_chunk_map_t *fm, int *min_chunkf);
static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
const H5D_chunk_map_t *fm, int *sum_chunkf);
+static herr_t H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
+ H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries);
+static herr_t H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info,
+ const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
+ H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries);
+static herr_t H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries,
+ size_t array_entry_size, void **gathered_array, size_t *gathered_array_num_entries,
+ int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *));
+static herr_t H5D__mpio_filtered_collective_write_type(
+ H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries,
+ MPI_Datatype *new_mem_type, hbool_t *mem_type_derived,
+ MPI_Datatype *new_file_type, hbool_t *file_type_derived);
+static herr_t H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry,
+ const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm);
+static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2);
+static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1,
+ const void *filtered_collective_io_info_entry2);
+static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1,
+ const void *filtered_collective_io_info_entry2);
/*********************/
@@ -142,7 +274,7 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
* Purpose: Checks if an direct I/O transfer is possible between memory and
* the file.
*
- * Return: Sauccess: Non-negative: TRUE or FALSE
+ * Return: Success: Non-negative: TRUE or FALSE
* Failure: Negative
*
* Programmer: Quincey Koziol
@@ -152,12 +284,11 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info,
*/
htri_t
H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
- const H5S_t *mem_space, const H5D_type_info_t *type_info,
- const H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+ const H5S_t *mem_space, const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist)
{
int local_cause = 0; /* Local reason(s) for breaking collective mode */
int global_cause = 0; /* Global reason(s) for breaking collective mode */
- htri_t ret_value; /* Return value */
+ htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@@ -206,11 +337,6 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
* use collective IO will defer until each chunk IO is reached.
*/
- /* Don't allow collective operations if filters need to be applied */
- if(io_info->dset->shared->layout.type == H5D_CHUNKED &&
- io_info->dset->shared->dcpl_cache.pline.nused > 0)
- local_cause |= H5D_MPIO_FILTERS;
-
/* Check for independent I/O */
if(local_cause & H5D_MPIO_SET_INDEPENDENT)
global_cause = local_cause;
@@ -300,6 +426,113 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5D__mpio_array_gatherv
+ *
+ * Purpose: Given an array, specified in local_array, by each processor
+ * calling this function, gathers each array into a single
+ * array which is then either gathered to the processor
+ * specified by root, when allgather is false, or is
+ * distributed back to all processors when allgather is true.
+ *
+ * The size of each entry and number of entries in the array
+ * contributed by an individual processor should be specified
+ * in array_entry_size and local_array_num_entries,
+ * respectively.
+ *
+ * The number of processors participating in the gather
+ * operation should be specified for nprocs.
+ *
+ * The MPI communicator to use should be specified for comm.
+ *
+ * If the sort_func argument is supplied, the array is sorted
+ * before the function returns.
+ *
+ * Note: if allgather is specified as true, root is ignored.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Sunday, April 9th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries,
+ size_t array_entry_size, void **_gathered_array, size_t *_gathered_array_num_entries,
+ int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *))
+{
+ size_t gathered_array_num_entries = 0; /* The size of the newly-constructed array */
+ size_t i;
+ void *gathered_array = NULL; /* The newly-constructed array returned to the caller */
+ int *receive_counts_array = NULL; /* Array containing number of entries each process is contributing */
+ int *displacements_array = NULL; /* Array of displacements where each process places its data in the final array */
+ int mpi_code;
+ int sendcount;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(_gathered_array);
+ HDassert(_gathered_array_num_entries);
+
+ /* Determine the size of the end result array */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_array_num_entries, &gathered_array_num_entries, 1, MPI_INT, MPI_SUM, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
+
+ /* If 0 entries resulted from the collective operation, no one is writing anything */
+ if (gathered_array_num_entries > 0) {
+ if (NULL == (gathered_array = H5MM_malloc(gathered_array_num_entries * array_entry_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate gathered array")
+
+ if (NULL == (receive_counts_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive counts array")
+
+ if (NULL == (displacements_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive displacements array")
+
+ /* Inform each process of how many entries each other process is contributing to the resulting array */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&local_array_num_entries, 1, MPI_INT, receive_counts_array, 1, MPI_INT, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* Multiply each receive count by the size of the array entry, since the data is sent as bytes */
+ for (i = 0; i < (size_t) nprocs; i++)
+ H5_CHECKED_ASSIGN(receive_counts_array[i], int, (size_t) receive_counts_array[i] * array_entry_size, size_t);
+
+ /* Set receive buffer offsets for MPI_Allgatherv */
+ displacements_array[0] = 0;
+ for (i = 1; i < (size_t) nprocs; i++)
+ displacements_array[i] = displacements_array[i - 1] + receive_counts_array[i - 1];
+
+ H5_CHECKED_ASSIGN(sendcount, int, local_array_num_entries * array_entry_size, size_t);
+
+ if (allgather) {
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgatherv(local_array, sendcount, MPI_BYTE,
+ gathered_array, receive_counts_array, displacements_array, MPI_BYTE, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code)
+ } /* end if */
+ else {
+ if (MPI_SUCCESS != (mpi_code = MPI_Gatherv(local_array, sendcount, MPI_BYTE,
+ gathered_array, receive_counts_array, displacements_array, MPI_BYTE, root, comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code)
+ } /* end else */
+
+ if (sort_func) HDqsort(gathered_array, gathered_array_num_entries, array_entry_size, sort_func);
+ } /* end if */
+
+ *_gathered_array = gathered_array;
+ *_gathered_array_num_entries = gathered_array_num_entries;
+
+done:
+ if (receive_counts_array)
+ H5MM_free(receive_counts_array);
+ if (displacements_array)
+ H5MM_free(displacements_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__mpio_array_gatherv() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__ioinfo_xfer_mode
*
* Purpose: Switch to between collective & independent MPI I/O
@@ -398,7 +631,7 @@ H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm,
FUNC_ENTER_STATIC
/* Get the number of chunks to perform I/O on */
- num_chunkf = H5SL_count(fm->sel_chunks);
+ H5_CHECKED_ASSIGN(num_chunkf, int, H5SL_count(fm->sel_chunks), size_t)
/* Determine the minimum # of chunks for all processes */
if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&num_chunkf, min_chunkf, 1, MPI_INT, MPI_MIN, io_info->comm)))
@@ -480,7 +713,7 @@ H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish shared collective MPI-IO")
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
/* Set the actual I/O mode property. internal_collective_io will not break to
@@ -527,7 +760,7 @@ H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't finish shared collective MPI-IO")
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
/* Set the actual I/O mode property. internal_collective_io will not break to
@@ -599,12 +832,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
HDassert(fm);
/* Obtain the data transfer properties */
- if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id)))
+ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
- /* Check the optional property list on what to do with collective chunk IO. */
+ /* Check the optional property list for the collective chunk IO optimization option */
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME, &chunk_opt_mode) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option")
+
if(H5FD_MPIO_CHUNK_ONE_IO == chunk_opt_mode)
io_option = H5D_ONE_LINK_CHUNK_IO; /*no opt*/
/* direct request to multi-chunk-io */
@@ -620,13 +854,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
if((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
- /* Get the chunk optimization option */
+ /* Get the chunk optimization option threshold */
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME, &one_link_chunk_io_threshold) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option")
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option threshold value")
/* step 1: choose an IO option */
/* If the average number of chunk per process is greater than a threshold, we will do one link chunked IO. */
- if((unsigned)sum_chunk / mpi_size >= one_link_chunk_io_threshold)
+ if((unsigned)sum_chunk / (unsigned)mpi_size >= one_link_chunk_io_threshold)
io_option = H5D_ONE_LINK_CHUNK_IO_MORE_OPT;
#ifdef H5_HAVE_INSTRUMENTED_LIBRARY
else
@@ -681,19 +915,46 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
#endif
/* step 2: Go ahead to do IO.*/
- if(H5D_ONE_LINK_CHUNK_IO == io_option || H5D_ONE_LINK_CHUNK_IO_MORE_OPT == io_option) {
- if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO")
- } /* end if */
- /* direct request to multi-chunk-io */
- else if(H5D_MULTI_CHUNK_IO == io_option) {
- if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
- } /* end if */
- else { /* multiple chunk IO via threshold */
- if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
- } /* end else */
+ switch (io_option) {
+ case H5D_ONE_LINK_CHUNK_IO:
+ case H5D_ONE_LINK_CHUNK_IO_MORE_OPT:
+ /* Check if there are any filters in the pipeline */
+ if(io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ /* For now, Multi-chunk IO must be forced for parallel filtered read,
+ * so that data can be unfiltered as it is received. There is significant
+ * complexity in unfiltering the data when it is read all at once into a
+ * single buffer.
+ */
+ if (io_info->op_type == H5D_IO_OP_READ) {
+ if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO")
+ } /* end if */
+ else {
+ if(H5D__link_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish filtered linked chunk MPI-IO")
+ } /* end else */
+ } /* end if */
+ else {
+ /* Perform unfiltered link chunk collective IO */
+ if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO")
+ } /* end else */
+ break;
+
+ case H5D_MULTI_CHUNK_IO: /* direct request to do multi-chunk IO */
+ default: /* multiple chunk IO via threshold */
+ /* Check if there are any filters in the pipeline */
+ if(io_info->dset->shared->dcpl_cache.pline.nused > 0) {
+ if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO")
+ } /* end if */
+ else {
+ /* Perform unfiltered multi chunk collective IO */
+ if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO")
+ } /* end else */
+ break;
+ } /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -1093,6 +1354,232 @@ if(H5DEBUG(D))
/*-------------------------------------------------------------------------
+ * Function: H5D__link_chunk_filtered_collective_io
+ *
+ * Purpose: Routine for one collective IO with one MPI derived datatype
+ * to link with all filtered chunks
+ *
+ * 1. Construct a list of selected chunks in the collective IO
+ * operation
+ * A. If any chunk is being written to by more than 1
+ * process, the process writing to the chunk which
+ * currently has the least amount of chunks assigned
+ * to it becomes the new owner (in the case of ties,
+ * the lowest MPI rank becomes the new owner)
+ * 2. If the operation is a write operation
+ * A. Loop through each chunk in the operation
+ * I. If this is not a full overwrite of the chunk
+ * a) Read the chunk from file and pass the chunk
+ * through the filter pipeline in reverse order
+ * (Unfilter the chunk)
+ * II. Update the chunk data with the modifications from
+ * the owning process
+ * III. Receive any modification data from other
+ * processes and update the chunk data with these
+ * modifications
+ * IV. Filter the chunk
+ * B. Contribute the modified chunks to an array gathered
+ * by all processes which contains the new sizes of
+ * every chunk modified in the collective IO operation
+ * C. All processes collectively re-allocate each chunk
+ * from the gathered array with their new sizes after
+ * the filter operation
+ * D. If this process has any chunks selected in the IO
+ * operation, create an MPI derived type for memory and
+ * file to write out the process' selected chunks to the
+ * file
+ * E. Perform the collective write
+ * F. All processes collectively re-insert each modified
+ * chunk from the gathered array into the chunk index
+ *
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Friday, Nov. 4th, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+{
+ H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */
+ H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */
+ H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_LINK_CHUNK; /* The actual chunk IO optimization mode */
+ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */
+ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */
+ MPI_Datatype mem_type = MPI_BYTE;
+ MPI_Datatype file_type = MPI_BYTE;
+ hbool_t mem_type_is_derived = FALSE;
+ hbool_t file_type_is_derived = FALSE;
+ size_t chunk_list_num_entries;
+ size_t collective_chunk_list_num_entries;
+ size_t *num_chunks_selected_array = NULL; /* Array of number of chunks selected on each process */
+ size_t i; /* Local index variable */
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(dx_plist);
+
+ /* Obtain the current rank of the process and the number of processes */
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ /* Set the actual-chunk-opt-mode property. */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property")
+
+ /* Set the actual-io-mode property.
+ * Link chunk filtered I/O does not break to independent, so can set right away
+ */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual io mode property")
+
+ /* Build a list of selected chunks in the collective io operation */
+ if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list")
+
+ if (io_info->op_type == H5D_IO_OP_WRITE) { /* Filtered collective write */
+ H5D_chk_idx_info_t index_info;
+ H5D_chunk_ud_t udata;
+ hsize_t mpi_buf_count;
+
+ /* Construct chunked index info */
+ index_info.f = io_info->dset->oloc.file;
+ index_info.dxpl_id = io_info->md_dxpl_id;
+ index_info.pline = &(io_info->dset->shared->dcpl_cache.pline);
+ index_info.layout = &(io_info->dset->shared->layout.u.chunk);
+ index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk);
+
+ /* Set up chunk information for insertion to chunk index */
+ udata.common.layout = index_info.layout;
+ udata.common.storage = index_info.storage;
+ udata.filter_mask = 0;
+
+ /* Iterate through all the chunks in the collective write operation,
+ * updating each chunk with the data modifications from other processes,
+ * then re-filtering the chunk.
+ */
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (mpi_rank == chunk_list[i].owners.new_owner)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry")
+
+ /* Gather the new chunk sizes to all processes for a collective reallocation
+ * of the chunks in the file.
+ */
+ if (H5D__mpio_array_gatherv(chunk_list, chunk_list_num_entries, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size,
+ true, 0, io_info->comm, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes")
+
+ /* Collectively re-allocate the modified chunks (from each process) in the file */
+ for (i = 0; i < collective_chunk_list_num_entries; i++) {
+ hbool_t insert;
+
+ if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[i].chunk_states.chunk_current,
+ &collective_chunk_list[i].chunk_states.new_chunk, &insert, collective_chunk_list[i].scaled) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+ } /* end for */
+
+ if (NULL == (num_chunks_selected_array = (size_t *) H5MM_malloc((size_t) mpi_size * sizeof(size_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, MPI_UNSIGNED_LONG_LONG, num_chunks_selected_array,
+ 1, MPI_UNSIGNED_LONG_LONG, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* If this process has any chunks selected, create a MPI type for collectively
+ * writing out the chunks to file. Otherwise, the process contributes to the
+ * collective write with a none type.
+ */
+ if (chunk_list_num_entries) {
+ size_t offset;
+
+ /* During the collective re-allocation of chunks in the file, the record for each
+ * chunk is only updated in the collective array, not in the local copy of chunks on each
+ * process. However, each process needs the updated chunk records so that they can create
+ * a MPI type for the collective write that will write to the chunk's possible new locations
+ * in the file instead of the old ones. This ugly hack seems to be the best solution to
+ * copy the information back to the local array and avoid having to modify the collective
+ * write type function in an ugly way so that it will accept the collective array instead
+ * of the local array. This works correctly because the array gather function guarantees
+ * that the chunk data in the collective array is ordered in blocks by rank.
+ */
+ for (i = 0, offset = 0; i < (size_t) mpi_rank; i++)
+ offset += num_chunks_selected_array[i];
+
+ HDmemcpy(chunk_list, &collective_chunk_list[offset], num_chunks_selected_array[mpi_rank] * sizeof(H5D_filtered_collective_io_info_t));
+
+ /* Create single MPI type encompassing each selection in the dataspace */
+ if (H5D__mpio_filtered_collective_write_type(chunk_list, chunk_list_num_entries,
+ &mem_type, &mem_type_is_derived, &file_type, &file_type_is_derived) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI link chunk I/O type")
+
+ /* Override the write buffer to point to the address of the first
+ * chunk data buffer
+ */
+ io_info->u.wbuf = chunk_list[0].buf;
+ } /* end if */
+
+ /* We have a single, complicated MPI datatype for both memory & file */
+ mpi_buf_count = (mem_type_is_derived && file_type_is_derived) ? (hsize_t) 1 : (hsize_t) 0;
+
+ /* Set up the base storage address for this operation */
+ ctg_store.contig.dset_addr = 0; /* Write address must be set to address 0 */
+ io_info->store = &ctg_store;
+
+ /* Perform I/O */
+ if (H5D__final_collective_io(io_info, type_info, mpi_buf_count, &file_type, &mem_type) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO")
+
+ /* Participate in the collective re-insertion of all chunks modified
+ * in this iteration into the chunk index
+ */
+ for (i = 0; i < collective_chunk_list_num_entries; i++) {
+ udata.chunk_block = collective_chunk_list[i].chunk_states.new_chunk;
+ udata.common.scaled = collective_chunk_list[i].scaled;
+ udata.chunk_idx = collective_chunk_list[i].index;
+
+ if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index")
+ } /* end for */
+ } /* end if */
+
+done:
+ /* Free resources used by a process which had some selection */
+ if (chunk_list) {
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (chunk_list[i].buf)
+ H5MM_free(chunk_list[i].buf);
+
+ H5MM_free(chunk_list);
+ } /* end if */
+
+ if (num_chunks_selected_array)
+ H5MM_free(num_chunks_selected_array);
+ if (collective_chunk_list)
+ H5MM_free(collective_chunk_list);
+
+ /* Free the MPI buf and file types, if they were derived */
+ if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type)))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+ if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type)))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__link_chunk_filtered_collective_io() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__multi_chunk_collective_io
*
* Purpose: To do IO per chunk according to IO mode(collective/independent/none)
@@ -1225,7 +1712,7 @@ if(H5DEBUG(D))
* to ease switching between to mixed I/O without checking the current
* value of the property. You can see the definition in H5Ppublic.h
*/
- actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE;
+ actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE);
} /* end if */
else {
@@ -1265,7 +1752,7 @@ if(H5DEBUG(D))
mspace = chunk_info->mspace;
/* Update the local variable tracking the dxpl's actual io mode. */
- actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT;
+ actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT);
} /* end if */
else {
fspace = mspace = NULL;
@@ -1306,6 +1793,314 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5D__multi_chunk_filtered_collective_io
+ *
+ * Purpose: To do filtered collective IO iteratively to save on memory.
+ * While link_chunk_filtered_collective_io will construct and
+ * work on a list of all of the chunks selected in the IO
+ * operation at once, this function works iteratively on a set
+ * of chunks at a time; at most one chunk per rank per
+ * iteration.
+ *
+ * 1. Construct a list of selected chunks in the collective IO
+ * operation
+ * A. If any chunk is being written to by more than 1
+ * process, the process writing to the chunk which
+ * currently has the least amount of chunks assigned
+ * to it becomes the new owner (in the case of ties,
+ * the lowest MPI rank becomes the new owner)
+ * 2. If the operation is a read operation
+ * A. Loop through each chunk in the operation
+ * I. Read the chunk from the file
+ * II. Unfilter the chunk
+ * III. Scatter the read chunk data to the user's buffer
+ * 3. If the operation is a write operation
+ * A. Loop through each chunk in the operation
+ * I. If this is not a full overwrite of the chunk
+ * a) Read the chunk from file and pass the chunk
+ * through the filter pipeline in reverse order
+ * (Unfilter the chunk)
+ * II. Update the chunk data with the modifications from
+ * the owning process
+ * III. Receive any modification data from other
+ * processes and update the chunk data with these
+ * modifications
+ * IV. Filter the chunk
+ * V. Contribute the chunk to an array gathered by
+ * all processes which contains every chunk
+ * modified in this iteration (up to one chunk
+ * per process, some processes may not have a
+ * selection/may have less chunks to work on than
+ * other processes)
+ * VI. All processes collectively re-allocate each
+ * chunk from the gathered array with their new
+ * sizes after the filter operation
+ * VII. Proceed with the collective write operation
+ * for the chunks modified on this iteration
+ * VIII. All processes collectively re-insert each
+ * chunk from the gathered array into the chunk
+ * index
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Friday, Dec. 2nd, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist)
+{
+ H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */
+ H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */
+ H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_MULTI_CHUNK; /* The actual chunk IO optimization mode */
+ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */
+ H5D_storage_t store; /* union of EFL and chunk pointer in file space */
+ H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */
+ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */
+ MPI_Datatype *file_type_array = NULL;
+ MPI_Datatype *mem_type_array = NULL;
+ hbool_t *file_type_is_derived_array = NULL;
+ hbool_t *mem_type_is_derived_array = NULL;
+ hbool_t *has_chunk_selected_array = NULL; /* Array of whether or not each process is contributing a chunk to each iteration */
+ size_t chunk_list_num_entries;
+ size_t collective_chunk_list_num_entries;
+ size_t i, j; /* Local index variable */
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(dx_plist);
+
+ /* Obtain the current rank of the process and the number of processes */
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ /* Set the actual chunk opt mode property */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property")
+
+ /* Set the actual_io_mode property.
+ * Multi chunk I/O does not break to independent, so can set right away
+ */
+ if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk io mode property")
+
+ /* Build a list of selected chunks in the collective IO operation */
+ if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list")
+
+ /* Set up contiguous I/O info object */
+ HDmemcpy(&ctg_io_info, io_info, sizeof(ctg_io_info));
+ ctg_io_info.store = &ctg_store;
+ ctg_io_info.layout_ops = *H5D_LOPS_CONTIG;
+
+ /* Initialize temporary contiguous storage info */
+ ctg_store.contig.dset_size = (hsize_t) io_info->dset->shared->layout.u.chunk.size;
+ ctg_store.contig.dset_addr = 0;
+
+ /* Set dataset storage for I/O info */
+ io_info->store = &store;
+
+ if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't process chunk entry")
+ } /* end if */
+ else { /* Filtered collective write */
+ H5D_chk_idx_info_t index_info;
+ H5D_chunk_ud_t udata;
+ size_t max_num_chunks;
+ hsize_t mpi_buf_count;
+
+ /* Construct chunked index info */
+ index_info.f = io_info->dset->oloc.file;
+ index_info.dxpl_id = io_info->md_dxpl_id;
+ index_info.pline = &(io_info->dset->shared->dcpl_cache.pline);
+ index_info.layout = &(io_info->dset->shared->layout.u.chunk);
+ index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk);
+
+ /* Set up chunk information for insertion to chunk index */
+ udata.common.layout = index_info.layout;
+ udata.common.storage = index_info.storage;
+ udata.filter_mask = 0;
+
+ /* Retrieve the maximum number of chunks being written among all processes */
+ if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks,
+ 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
+
+ /* If no one is writing anything at all, end the operation */
+ if (!(max_num_chunks > 0)) HGOTO_DONE(SUCCEED);
+
+ /* Allocate arrays for storing MPI file and mem types and whether or not the
+ * types were derived.
+ */
+ if (NULL == (file_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type array")
+
+ if (NULL == (file_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type is derived array")
+
+ if (NULL == (mem_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type array")
+
+ if (NULL == (mem_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type is derived array")
+
+ /* Iterate over the max number of chunks among all processes, as this process could
+ * have no chunks left to work on, but it still needs to participate in the collective
+ * re-allocation and re-insertion of chunks modified by other processes.
+ */
+ for (i = 0; i < max_num_chunks; i++) {
+ /* Check if this process has a chunk to work on for this iteration */
+ hbool_t have_chunk_to_process = (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].owners.new_owner);
+
+ if (have_chunk_to_process)
+ if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry")
+
+ /* Gather the new chunk sizes to all processes for a collective re-allocation
+ * of the chunks in the file
+ */
+ if (H5D__mpio_array_gatherv(&chunk_list[i], have_chunk_to_process ? 1 : 0, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size,
+ true, 0, io_info->comm, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes")
+
+ /* Participate in the collective re-allocation of all chunks modified
+ * in this iteration.
+ */
+ for (j = 0; j < collective_chunk_list_num_entries; j++) {
+ hbool_t insert = FALSE;
+
+ if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[j].chunk_states.chunk_current,
+ &collective_chunk_list[j].chunk_states.new_chunk, &insert, chunk_list[j].scaled) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+ } /* end for */
+
+ if (NULL == (has_chunk_selected_array = (hbool_t *) H5MM_malloc((size_t) mpi_size * sizeof(hbool_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&have_chunk_to_process, 1, MPI_C_BOOL, has_chunk_selected_array,
+ 1, MPI_C_BOOL, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code)
+
+ /* If this process has a chunk to work on, create a MPI type for the
+ * memory and file for writing out the chunk
+ */
+ if (have_chunk_to_process) {
+ size_t offset;
+ int mpi_type_count;
+
+ for (j = 0, offset = 0; j < (size_t) mpi_rank; j++)
+ offset += has_chunk_selected_array[j];
+
+ /* Collect the new chunk info back to the local copy, since only the record in the
+ * collective array gets updated by the chunk re-allocation */
+ HDmemcpy(&chunk_list[i].chunk_states.new_chunk, &collective_chunk_list[offset].chunk_states.new_chunk, sizeof(chunk_list[i].chunk_states.new_chunk));
+
+ H5_CHECKED_ASSIGN(mpi_type_count, int, chunk_list[i].chunk_states.new_chunk.length, hsize_t);
+
+ /* Create MPI memory type for writing to chunk */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &mem_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
+ if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ mem_type_is_derived_array[i] = TRUE;
+
+ /* Create MPI file type for writing to chunk */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &file_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
+ if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type_array[i])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ file_type_is_derived_array[i] = TRUE;
+
+ mpi_buf_count = 1;
+
+ /* Set up the base storage address for this operation */
+ ctg_store.contig.dset_addr = chunk_list[i].chunk_states.new_chunk.offset;
+
+ /* Override the write buffer to point to the address of the
+ * chunk data buffer
+ */
+ ctg_io_info.u.wbuf = chunk_list[i].buf;
+ } /* end if */
+ else {
+ mem_type_array[i] = file_type_array[i] = MPI_BYTE;
+ mpi_buf_count = 0;
+ } /* end else */
+
+ /* Perform the I/O */
+ if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, &file_type_array[i], &mem_type_array[i]) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO")
+
+ /* Participate in the collective re-insertion of all chunks modified
+ * in this iteration into the chunk index
+ */
+ for (j = 0; j < collective_chunk_list_num_entries; j++) {
+ udata.chunk_block = collective_chunk_list[j].chunk_states.new_chunk;
+ udata.common.scaled = collective_chunk_list[j].scaled;
+ udata.chunk_idx = collective_chunk_list[j].index;
+
+ if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index")
+ } /* end for */
+
+ if (collective_chunk_list){
+ H5MM_free(collective_chunk_list);
+ collective_chunk_list = NULL;
+ } /* end if */
+ if (has_chunk_selected_array){
+ H5MM_free(has_chunk_selected_array);
+ has_chunk_selected_array = NULL;
+ } /* end if */
+ } /* end for */
+
+ /* Free the MPI file and memory types, if they were derived */
+ for (i = 0; i < max_num_chunks; i++) {
+ if (file_type_is_derived_array[i])
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type_array[i])))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+
+ if (mem_type_is_derived_array[i])
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type_array[i])))
+ HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code)
+ } /* end for */
+ } /* end else */
+
+done:
+ if (chunk_list) {
+ for (i = 0; i < chunk_list_num_entries; i++)
+ if (chunk_list[i].buf)
+ H5MM_free(chunk_list[i].buf);
+
+ H5MM_free(chunk_list);
+ } /* end if */
+
+ if (collective_chunk_list)
+ H5MM_free(collective_chunk_list);
+ if (file_type_array)
+ H5MM_free(file_type_array);
+ if (mem_type_array)
+ H5MM_free(mem_type_array);
+ if (file_type_is_derived_array)
+ H5MM_free(file_type_is_derived_array);
+ if (mem_type_is_derived_array)
+ H5MM_free(mem_type_is_derived_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__multi_chunk_filtered_collective_io() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__inter_collective_io
*
* Purpose: Routine for the shared part of collective IO between multiple chunk
@@ -1472,7 +2267,7 @@ if(H5DEBUG(D))
static int
H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2)
{
- haddr_t addr1, addr2;
+ haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF;
FUNC_ENTER_STATIC_NOERR
@@ -1484,6 +2279,67 @@ H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2)
/*-------------------------------------------------------------------------
+ * Function: H5D__cmp_filtered_collective_io_info_entry
+ *
+ * Purpose: Routine to compare filtered collective chunk io info
+ * entries
+ *
+ * Description: Callback for qsort() to compare filtered collective chunk
+ * io info entries
+ *
+ * Return: -1, 0, 1
+ *
+ * Programmer: Jordan Henderson
+ * Wednesday, Nov. 30th, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2)
+{
+ haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ addr1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->chunk_states.new_chunk.offset;
+ addr2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->chunk_states.new_chunk.offset;
+
+ FUNC_LEAVE_NOAPI(H5F_addr_cmp(addr1, addr2))
+} /* end H5D__cmp_filtered_collective_io_info_entry() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__cmp_filtered_collective_io_info_entry_owner
+ *
+ * Purpose: Routine to compare filtered collective chunk io info
+ * entries's original owner fields
+ *
+ * Description: Callback for qsort() to compare filtered collective chunk
+ * io info entries's original owner fields
+ *
+ * Return: The difference between the two
+ * H5D_filtered_collective_io_info_t's original owner fields
+ *
+ * Programmer: Jordan Henderson
+ * Monday, Apr. 10th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2)
+{
+ int owner1 = -1, owner2 = -1;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ owner1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->owners.original_owner;
+ owner2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->owners.original_owner;
+
+ FUNC_LEAVE_NOAPI(owner1 - owner2)
+} /* end H5D__cmp_filtered_collective_io_info_entry_owner() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5D__sort_chunk
*
* Purpose: Routine to sort chunks in increasing order of chunk address
@@ -1557,7 +2413,7 @@ if(H5DEBUG(D))
HDfprintf(H5DEBUG(D), "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL\n");
#endif
/* Allocate array for chunk addresses */
- if(NULL == (total_chunk_addr_array = H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks)))
+ if(NULL == (total_chunk_addr_array = (haddr_t *)H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory chunk address array")
/* Retrieve all the chunk addresses with process 0 */
@@ -1581,7 +2437,7 @@ if(H5DEBUG(D))
/* Iterate over all chunks for this process */
while(chunk_node) {
- if(NULL == (chunk_info = H5SL_item(chunk_node)))
+ if(NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node)))
HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL,"couldn't get chunk info from skipped list")
if(many_chunk_opt == H5D_OBTAIN_ONE_CHUNK_ADDR_IND) {
@@ -1666,7 +2522,7 @@ static herr_t
H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
H5P_genplist_t *dx_plist, uint8_t assign_io_mode[], haddr_t chunk_addr[])
{
- int total_chunks;
+ size_t total_chunks;
unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk;
uint8_t* io_mode_info = NULL;
uint8_t* recv_io_mode_info = NULL;
@@ -1676,7 +2532,8 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
H5D_chunk_info_t* chunk_info;
int mpi_size, mpi_rank;
MPI_Comm comm;
- int ic, root;
+ int root;
+ size_t ic;
int mpi_code;
#ifdef H5_HAVE_INSTRUMENTED_LIBRARY
int new_value;
@@ -1697,7 +2554,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
/* Setup parameters */
- H5_CHECKED_ASSIGN(total_chunks, int, fm->layout->u.chunk.nchunks, hsize_t);
+ H5_CHECKED_ASSIGN(total_chunks, size_t, fm->layout->u.chunk.nchunks, hsize_t);
if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME, &percent_nproc_per_chunk) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get percent nproc per chunk")
/* if ratio is 0, perform collective io */
@@ -1709,39 +2566,42 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
HGOTO_DONE(SUCCEED)
} /* end if */
- threshold_nproc_per_chunk = mpi_size * percent_nproc_per_chunk/100;
+
+ threshold_nproc_per_chunk = (unsigned)mpi_size * percent_nproc_per_chunk/100;
/* Allocate memory */
if(NULL == (io_mode_info = (uint8_t *)H5MM_calloc(total_chunks)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate I/O mode info buffer")
- if(NULL == (mergebuf = H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks)))
+ if(NULL == (mergebuf = (uint8_t *)H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mergebuf buffer")
tempbuf = mergebuf + total_chunks;
if(mpi_rank == root)
- if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * mpi_size)))
+ if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * (size_t)mpi_size)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate recv I/O mode info buffer")
/* Obtain the regularity and selection information for all chunks in this process. */
chunk_node = H5SL_first(fm->sel_chunks);
while(chunk_node) {
- chunk_info = H5SL_item(chunk_node);
+ chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node);
- io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */
+ io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */
chunk_node = H5SL_next(chunk_node);
} /* end while */
/* Gather all the information */
- if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, total_chunks, MPI_BYTE, recv_io_mode_info, total_chunks, MPI_BYTE, root, comm)))
+ H5_CHECK_OVERFLOW(total_chunks, size_t, int)
+ if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, (int)total_chunks, MPI_BYTE,
+ recv_io_mode_info, (int)total_chunks, MPI_BYTE, root, comm)))
HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code)
/* Calculate the mode for IO(collective, independent or none) at root process */
if(mpi_rank == root) {
- int nproc;
- int* nproc_per_chunk;
+ size_t nproc;
+ unsigned* nproc_per_chunk;
/* pre-computing: calculate number of processes and
regularity of the selection occupied in each chunk */
- if(NULL == (nproc_per_chunk = (int*)H5MM_calloc(total_chunks * sizeof(int))))
+ if(NULL == (nproc_per_chunk = (unsigned*)H5MM_calloc(total_chunks * sizeof(unsigned))))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate nproc_per_chunk buffer")
/* calculating the chunk address */
@@ -1751,7 +2611,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm,
} /* end if */
/* checking for number of process per chunk and regularity of the selection*/
- for(nproc = 0; nproc < mpi_size; nproc++) {
+ for(nproc = 0; nproc < (size_t)mpi_size; nproc++) {
uint8_t *tmp_recv_io_mode_info = recv_io_mode_info + (nproc * total_chunks);
/* Calculate the number of process per chunk and adding irregular selection option */
@@ -1835,5 +2695,712 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__obtain_mpio_mode() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__construct_filtered_io_info_list
+ *
+ * Purpose: Constructs a list of entries which contain the necessary
+ * information for inter-process communication when performing
+ * collective io on filtered chunks. This list is used by
+ * each process when performing I/O on locally selected chunks
+ * and also in operations that must be collectively done
+ * on every chunk, such as chunk re-allocation, insertion of
+ * chunks into the chunk index, etc.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Tuesday, January 10th, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries)
+{
+ H5D_filtered_collective_io_info_t *local_info_array = NULL; /* The list of initially selected chunks for this process */
+ size_t num_chunks_selected;
+ size_t i;
+ int mpi_rank;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(chunk_list);
+ HDassert(num_entries);
+ HDassert(TRUE == H5P_isa_class(io_info->raw_dxpl_id, H5P_DATASET_XFER));
+
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+
+ /* Each process builds a local list of the chunks they have selected */
+ if ((num_chunks_selected = H5SL_count(fm->sel_chunks))) {
+ H5D_chunk_info_t *chunk_info;
+ H5D_chunk_ud_t udata;
+ H5SL_node_t *chunk_node;
+ hssize_t select_npoints;
+ hssize_t chunk_npoints;
+
+ if (NULL == (local_info_array = (H5D_filtered_collective_io_info_t *) H5MM_malloc(num_chunks_selected * sizeof(H5D_filtered_collective_io_info_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate local io info array buffer")
+
+ chunk_node = H5SL_first(fm->sel_chunks);
+ for (i = 0; chunk_node; i++) {
+ chunk_info = (H5D_chunk_info_t *) H5SL_item(chunk_node);
+
+ /* Obtain this chunk's address */
+ if (H5D__chunk_lookup(io_info->dset, io_info->md_dxpl_id, chunk_info->scaled, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+
+ local_info_array[i].index = chunk_info->index;
+ local_info_array[i].chunk_states.chunk_current = local_info_array[i].chunk_states.new_chunk = udata.chunk_block;
+ local_info_array[i].num_writers = 0;
+ local_info_array[i].owners.original_owner = local_info_array[i].owners.new_owner = mpi_rank;
+ local_info_array[i].buf = NULL;
+
+ local_info_array[i].async_info.num_receive_requests = 0;
+ local_info_array[i].async_info.receive_buffer_array = NULL;
+ local_info_array[i].async_info.receive_requests_array = NULL;
+
+ HDmemcpy(local_info_array[i].scaled, chunk_info->scaled, sizeof(chunk_info->scaled));
+
+ if ((select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ local_info_array[i].io_size = (size_t) select_npoints * type_info->src_type_size;
+
+ /* Currently the full overwrite status of a chunk is only obtained on a per-process
+ * basis. This means that if the total selection in the chunk, as determined by the combination
+ * of selections of all of the processes interested in the chunk, covers the entire chunk,
+ * the performance optimization of not reading the chunk from the file is still valid, but
+ * is not applied in the current implementation. Something like an appropriately placed
+ * MPI_Allreduce or a running total of the number of chunk points selected during chunk
+ * redistribution should suffice for implementing this case - JTH.
+ */
+ if ((chunk_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ local_info_array[i].full_overwrite =
+ (local_info_array[i].io_size >= (hsize_t) chunk_npoints * type_info->dst_type_size) ? TRUE : FALSE;
+
+ chunk_node = H5SL_next(chunk_node);
+ } /* end for */
+ } /* end if */
+
+ /* Redistribute shared chunks to new owners as necessary */
+ if (io_info->op_type == H5D_IO_OP_WRITE)
+ if (H5D__chunk_redistribute_shared_chunks(io_info, type_info, fm, local_info_array, &num_chunks_selected) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks")
+
+ *chunk_list = local_info_array;
+ *num_entries = num_chunks_selected;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__construct_filtered_io_info_list() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__chunk_redistribute_shared_chunks
+ *
+ * Purpose: When performing a collective write on a Dataset with
+ * filters applied, this function is used to redistribute any
+ * chunks which are selected by more than one process, so as
+ * to preserve file integrity after the write by ensuring
+ * that any shared chunks are only modified by one process.
+ *
+ * The current implementation follows this 3-phase process:
+ *
+ * - Collect everyone's list of chunks into one large list,
+ * sort the list in increasing order of chunk offset in the
+ * file and hand the list off to rank 0
+ *
+ * - Rank 0 scans the list looking for matching runs of chunk
+ * offset in the file (corresponding to a shared chunk which
+ * has been selected by more than one rank in the I/O
+ * operation) and for each shared chunk, it redistributes
+ * the chunk to the process writing to the chunk which
+ * currently has the least amount of chunks assigned to it
+ * by modifying the "new_owner" field in each of the list
+ * entries corresponding to that chunk
+ *
+ * - After the chunks have been redistributed, rank 0 re-sorts
+ * the list in order of previous owner so that each rank
+ * will get back exactly the array that they contributed to
+ * the redistribution operation, with the "new_owner" field
+ * of each chunk they are modifying having possibly been
+ * modified. Rank 0 then scatters each segment of the list
+ * back to its corresponding rank
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Monday, May 1, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
+ const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries)
+{
+ H5D_filtered_collective_io_info_t *shared_chunks_info_array = NULL; /* The list of all chunks selected in the operation by all processes */
+ H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__gather_mem */
+ unsigned char **mod_data = NULL; /* Array of chunk modification data buffers sent by a process to new chunk owners */
+ MPI_Request *send_requests = NULL; /* Array of MPI_Isend chunk modification data send requests */
+ MPI_Status *send_statuses = NULL; /* Array of MPI_Isend chunk modification send statuses */
+ hbool_t mem_iter_init = FALSE;
+ size_t shared_chunks_info_array_num_entries = 0;
+ size_t num_send_requests = 0;
+ size_t *num_assigned_chunks_array = NULL;
+ size_t i, last_assigned_idx;
+ int *send_counts = NULL;
+ int *send_displacements = NULL;
+ int scatter_recvcount_int;
+ int mpi_rank, mpi_size, mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+ HDassert(local_chunk_array_num_entries);
+
+ if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank")
+ if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size")
+
+ if (*local_chunk_array_num_entries)
+ if (NULL == (send_requests = (MPI_Request *) H5MM_malloc(*local_chunk_array_num_entries * sizeof(MPI_Request))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests buffer")
+
+ if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator")
+
+ /* Gather every rank's list of chunks to rank 0 to allow it to perform the redistribution operation. After this
+ * call, the gathered list will initially be sorted in increasing order of chunk offset in the file.
+ */
+ if (H5D__mpio_array_gatherv(local_chunk_array, *local_chunk_array_num_entries, sizeof(H5D_filtered_collective_io_info_t),
+ (void **) &shared_chunks_info_array, &shared_chunks_info_array_num_entries, mpi_size,
+ false, 0, io_info->comm, H5D__cmp_filtered_collective_io_info_entry) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather array")
+
+ /* Rank 0 redistributes any shared chunks to new owners as necessary */
+ if (mpi_rank == 0) {
+ if (NULL == (send_counts = (int *) H5MM_calloc((size_t) mpi_size * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send counts buffer")
+
+ if (NULL == (send_displacements = (int *) H5MM_malloc((size_t) mpi_size * sizeof(int))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send displacements buffer")
+
+ if (NULL == (num_assigned_chunks_array = (size_t *) H5MM_calloc((size_t) mpi_size * sizeof(size_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate number of assigned chunks array")
+
+ for (i = 0; i < shared_chunks_info_array_num_entries;) {
+ H5D_filtered_collective_io_info_t chunk_entry;
+ haddr_t last_seen_addr = shared_chunks_info_array[i].chunk_states.chunk_current.offset;
+ size_t set_begin_index = i;
+ size_t num_writers = 0;
+ int new_chunk_owner = shared_chunks_info_array[i].owners.original_owner;
+
+ /* Process each set of duplicate entries caused by another process writing to the same chunk */
+ do {
+ chunk_entry = shared_chunks_info_array[i];
+
+ send_counts[chunk_entry.owners.original_owner] += (int) sizeof(chunk_entry);
+
+ /* The new owner of the chunk is determined by the process
+ * writing to the chunk which currently has the least amount
+ * of chunks assigned to it
+ */
+ if (num_assigned_chunks_array[chunk_entry.owners.original_owner] < num_assigned_chunks_array[new_chunk_owner])
+ new_chunk_owner = chunk_entry.owners.original_owner;
+
+ num_writers++;
+ } while (++i < shared_chunks_info_array_num_entries && shared_chunks_info_array[i].chunk_states.chunk_current.offset == last_seen_addr);
+
+ /* Set all of the chunk entries' "new_owner" fields */
+ for (; set_begin_index < i; set_begin_index++) {
+ shared_chunks_info_array[set_begin_index].owners.new_owner = new_chunk_owner;
+ shared_chunks_info_array[set_begin_index].num_writers = num_writers;
+ } /* end for */
+
+ num_assigned_chunks_array[new_chunk_owner]++;
+ } /* end for */
+
+ /* Sort the new list in order of previous owner so that each original owner of a chunk
+ * entry gets that entry back, with the possibly newly-modified "new_owner" field
+ */
+ HDqsort(shared_chunks_info_array, shared_chunks_info_array_num_entries,
+ sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry_owner);
+
+ send_displacements[0] = 0;
+ for (i = 1; i < (size_t) mpi_size; i++)
+ send_displacements[i] = send_displacements[i - 1] + send_counts[i - 1];
+ } /* end if */
+
+ /* Scatter the segments of the list back to each process */
+ H5_CHECKED_ASSIGN(scatter_recvcount_int, int, *local_chunk_array_num_entries * sizeof(H5D_filtered_collective_io_info_t), size_t);
+ if (MPI_SUCCESS != (mpi_code = MPI_Scatterv(shared_chunks_info_array, send_counts, send_displacements,
+ MPI_BYTE, local_chunk_array, scatter_recvcount_int, MPI_BYTE, 0, io_info->comm)))
+ HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code)
+
+ if (shared_chunks_info_array) {
+ H5MM_free(shared_chunks_info_array);
+ shared_chunks_info_array = NULL;
+ } /* end if */
+
+ /* Now that the chunks have been redistributed, each process must send its modification data
+ * to the new owners of any of the chunks it previously possessed. Accordingly, each process
+ * must also issue asynchronous receives for any messages it may receive for each of the
+ * chunks it is assigned, in order to avoid potential deadlocking issues.
+ */
+ if (*local_chunk_array_num_entries)
+ if (NULL == (mod_data = (unsigned char **) H5MM_malloc(*local_chunk_array_num_entries * sizeof(unsigned char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data buffer array")
+
+ for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) {
+ H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i];
+
+ if (mpi_rank != chunk_entry->owners.new_owner) {
+ H5D_chunk_info_t *chunk_info = NULL;
+ unsigned char *mod_data_p = NULL;
+ hssize_t iter_nelmts;
+ size_t mod_data_size;
+
+ /* Look up the chunk and get its file and memory dataspaces */
+ if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list")
+
+ /* Determine size of serialized chunk file dataspace, plus the size of
+ * the data being written
+ */
+ if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to get encoded dataspace size")
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ mod_data_size += (size_t) iter_nelmts * type_info->src_type_size;
+
+ if (NULL == (mod_data[num_send_requests] = (unsigned char *) H5MM_malloc(mod_data_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk modification send buffer")
+
+ /* Serialize the chunk's file dataspace into the buffer */
+ mod_data_p = mod_data[num_send_requests];
+ if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace")
+
+ /* Intialize iterator for memory selection */
+ if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ /* Collect the modification data into the buffer */
+ if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, mod_data_p))
+ HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer")
+
+ /* Send modification data to new owner */
+ H5_CHECK_OVERFLOW(mod_data_size, size_t, int)
+ H5_CHECK_OVERFLOW(chunk_entry->index, hsize_t, int)
+ if (MPI_SUCCESS != (mpi_code = MPI_Isend(mod_data[num_send_requests], (int) mod_data_size, MPI_BYTE,
+ chunk_entry->owners.new_owner, (int) chunk_entry->index, io_info->comm, &send_requests[num_send_requests])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Isend failed", mpi_code)
+
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator")
+ mem_iter_init = FALSE;
+
+ num_send_requests++;
+ } /* end if */
+ else {
+ /* Allocate all necessary buffers for an asynchronous receive operation */
+ if (chunk_entry->num_writers > 1) {
+ MPI_Message message;
+ MPI_Status status;
+ size_t j;
+
+ chunk_entry->async_info.num_receive_requests = (int) chunk_entry->num_writers - 1;
+ if (NULL == (chunk_entry->async_info.receive_requests_array = (MPI_Request *) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(MPI_Request))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async requests array")
+
+ if (NULL == (chunk_entry->async_info.receive_buffer_array = (unsigned char **) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(unsigned char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async receive buffers")
+
+ for (j = 0; j < chunk_entry->num_writers - 1; j++) {
+ int count = 0;
+
+ /* Probe for a particular message from any process, removing that message
+ * from the receive queue in the process and allocating that much memory
+ * for the asynchronous receive
+ */
+ if (MPI_SUCCESS != (mpi_code = MPI_Mprobe(MPI_ANY_SOURCE, (int) chunk_entry->index, io_info->comm, &message, &status)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Mprobe failed", mpi_code)
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_count(&status, MPI_BYTE, &count)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mpi_code)
+
+ HDassert(count >= 0);
+ if (NULL == (chunk_entry->async_info.receive_buffer_array[j] = (unsigned char *) H5MM_malloc((size_t) count * sizeof(char *))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data receive buffer")
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Imrecv(chunk_entry->async_info.receive_buffer_array[j], count, MPI_BYTE,
+ &message, &chunk_entry->async_info.receive_requests_array[j])))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Imrecv failed", mpi_code)
+ } /* end for */
+ } /* end if */
+
+ local_chunk_array[last_assigned_idx++] = local_chunk_array[i];
+ } /* end else */
+ } /* end for */
+
+ *local_chunk_array_num_entries = last_assigned_idx;
+
+ /* Wait for all async send requests to complete before returning */
+ if (num_send_requests) {
+ if (NULL == (send_statuses = (MPI_Status *) H5MM_malloc(num_send_requests * sizeof(MPI_Status))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send statuses buffer")
+
+ H5_CHECK_OVERFLOW(num_send_requests, size_t, int);
+ if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int) num_send_requests, send_requests, send_statuses)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code)
+ } /* end if */
+
+done:
+ /* Now that all async send requests have completed, free up the send
+ * buffers used in the async operations
+ */
+ for (i = 0; i < num_send_requests; i++) {
+ if (mod_data[i])
+ H5MM_free(mod_data[i]);
+ } /* end for */
+
+ if (send_requests)
+ H5MM_free(send_requests);
+ if (send_statuses)
+ H5MM_free(send_statuses);
+ if (send_counts)
+ H5MM_free(send_counts);
+ if (send_displacements)
+ H5MM_free(send_displacements);
+ if (mod_data)
+ H5MM_free(mod_data);
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ if (mem_iter)
+ H5MM_free(mem_iter);
+ if (num_assigned_chunks_array)
+ H5MM_free(num_assigned_chunks_array);
+ if (shared_chunks_info_array)
+ H5MM_free(shared_chunks_info_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__chunk_redistribute_shared_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__mpio_filtered_collective_write_type
+ *
+ * Purpose: Constructs a MPI derived datatype for both the memory and
+ * the file for a collective write of filtered chunks. The
+ * datatype contains the offsets in the file and the locations
+ * of the filtered chunk data buffers.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Tuesday, November 22, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list,
+ size_t num_entries, MPI_Datatype *new_mem_type, hbool_t *mem_type_derived,
+ MPI_Datatype *new_file_type, hbool_t *file_type_derived)
+{
+ MPI_Aint *write_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */
+ MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */
+ int *length_array = NULL; /* Filtered Chunk lengths */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(chunk_list);
+ HDassert(new_mem_type);
+ HDassert(mem_type_derived);
+ HDassert(new_file_type);
+ HDassert(file_type_derived);
+
+ if (num_entries > 0) {
+ size_t i;
+ int mpi_code;
+ void *base_buf;
+
+ H5_CHECK_OVERFLOW(num_entries, size_t, int);
+
+ /* Allocate arrays */
+ if (NULL == (length_array = (int *) H5MM_malloc((size_t) num_entries * sizeof(int))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write length array")
+ if (NULL == (write_buf_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write buf length array")
+ if (NULL == (file_offset_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for collective write offset array")
+
+ /* Ensure the list is sorted in ascending order of offset in the file */
+ HDqsort(chunk_list, num_entries, sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry);
+
+ base_buf = chunk_list[0].buf;
+ for (i = 0; i < num_entries; i++) {
+ /* Set up the offset in the file, the length of the chunk data, and the relative
+ * displacement of the chunk data write buffer
+ */
+ file_offset_array[i] = (MPI_Aint) chunk_list[i].chunk_states.new_chunk.offset;
+ length_array[i] = (int) chunk_list[i].chunk_states.new_chunk.length;
+ write_buf_array[i] = (MPI_Aint) chunk_list[i].buf - (MPI_Aint) base_buf;
+ } /* end for */
+
+ /* Create memory MPI type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, write_buf_array, MPI_BYTE, new_mem_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
+ *mem_type_derived = TRUE;
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+
+ /* Create file MPI type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, file_offset_array, MPI_BYTE, new_file_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
+ *file_type_derived = TRUE;
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
+ } /* end if */
+
+done:
+ if (write_buf_array)
+ H5MM_free(write_buf_array);
+ if (file_offset_array)
+ H5MM_free(file_offset_array);
+ if (length_array)
+ H5MM_free(length_array);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__mpio_filtered_collective_write_type() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__filtered_collective_chunk_entry_io
+ *
+ * Purpose: Given an entry for a filtered chunk, performs the necessary
+ * steps for updating the chunk data during a collective
+ * write, or for reading the chunk from file during a
+ * collective read.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Jordan Henderson
+ * Wednesday, January 18, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry,
+ const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm)
+{
+ H5D_chunk_info_t *chunk_info = NULL;
+ H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__scatter_mem/H5D__gather_mem */
+ unsigned char *mod_data = NULL; /* Chunk modification data sent by a process to a chunk's owner */
+ unsigned filter_mask = 0;
+ hssize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */
+ hssize_t extent_npoints;
+ hsize_t true_chunk_size;
+ hbool_t mem_iter_init = FALSE;
+ size_t buf_size;
+ size_t i;
+ H5S_t *dataspace = NULL; /* Other process' dataspace for the chunk */
+ void *tmp_gath_buf = NULL; /* Temporary gather buffer for owner of the chunk to gather into from
+ application write buffer before scattering out to the chunk data buffer */
+ int mpi_code;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(chunk_entry);
+ HDassert(io_info);
+ HDassert(type_info);
+ HDassert(fm);
+
+ /* Look up the chunk and get its file and memory dataspaces */
+ if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list")
+
+ if ((extent_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+ true_chunk_size = (hsize_t) extent_npoints * type_info->src_type_size;
+
+ /* If the size of the filtered chunk is larger than the number of points in the
+ * chunk file space extent times the datatype size, allocate enough space to hold the
+ * whole filtered chunk. Otherwise, allocate a buffer equal to the size of the
+ * chunk so that the unfiltering operation doesn't have to grow the buffer.
+ */
+ buf_size = MAX(chunk_entry->chunk_states.chunk_current.length, true_chunk_size);
+
+ if (NULL == (chunk_entry->buf = H5MM_malloc(buf_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer")
+
+ /* If this is not a full chunk overwrite or this is a read operation, the chunk must be
+ * read from the file and unfiltered.
+ */
+ if (!chunk_entry->full_overwrite || io_info->op_type == H5D_IO_OP_READ) {
+ chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length;
+
+ /* Currently, these chunk reads are done independently and will likely
+ * cause issues with collective metadata reads enabled. In the future,
+ * this should be refactored to use collective chunk reads - JTH */
+ if (H5F_block_read(io_info->dset->oloc.file, H5FD_MEM_DRAW, chunk_entry->chunk_states.chunk_current.offset,
+ chunk_entry->chunk_states.new_chunk.length, H5AC_rawdata_dxpl_id, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk")
+
+ if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, &filter_mask,
+ io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb,
+ (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying")
+ } /* end if */
+ else {
+ chunk_entry->chunk_states.new_chunk.length = true_chunk_size;
+ } /* end else */
+
+ /* Initialize iterator for memory selection */
+ if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t))))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator")
+
+ if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* If this is a read operation, scatter the read chunk data to the user's buffer.
+ *
+ * If this is a write operation, update the chunk data buffer with the modifications
+ * from the current process, then apply any modifications from other processes. Finally,
+ * filter the newly-updated chunk.
+ */
+ switch (io_info->op_type) {
+ case H5D_IO_OP_READ:
+ if (H5D__scatter_mem(chunk_entry->buf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, io_info->u.rbuf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to read buffer")
+ break;
+
+ case H5D_IO_OP_WRITE:
+ if (NULL == (tmp_gath_buf = H5MM_malloc((hsize_t) iter_nelmts * type_info->src_type_size)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer")
+
+ /* Gather modification data from the application write buffer into a temporary buffer */
+ if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, tmp_gath_buf))
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't gather from write buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+
+ /* Initialize iterator for file selection */
+ if (H5S_select_iter_init(mem_iter, chunk_info->fspace, type_info->dst_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize file selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* Scatter the owner's modification data into the chunk data buffer according to
+ * the file space.
+ */
+ if (H5D__scatter_mem(tmp_gath_buf, chunk_info->fspace, mem_iter,
+ (size_t) iter_nelmts, io_info->dxpl_cache, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to chunk data buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Waitall(chunk_entry->async_info.num_receive_requests,
+ chunk_entry->async_info.receive_requests_array, MPI_STATUSES_IGNORE)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code)
+
+ /* For each asynchronous receive call previously posted, receive the chunk modification
+ * buffer from another rank and update the chunk data
+ */
+ for (i = 0; i < (size_t) chunk_entry->async_info.num_receive_requests; i++) {
+ const unsigned char *mod_data_p;
+
+ /* Decode the process' chunk file dataspace */
+ mod_data_p = chunk_entry->async_info.receive_buffer_array[i];
+ if (NULL == (dataspace = H5S_decode(&mod_data_p)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace")
+
+ if (H5S_select_iter_init(mem_iter, dataspace, type_info->dst_type_size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information")
+ mem_iter_init = TRUE;
+
+ if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid")
+
+ /* Update the chunk data with the received modification data */
+ if (H5D__scatter_mem(mod_data_p, dataspace, mem_iter, (size_t) iter_nelmts,
+ io_info->dxpl_cache, chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer")
+
+ if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ mem_iter_init = FALSE;
+ if (dataspace) {
+ if (H5S_close(dataspace) < 0)
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace")
+ dataspace = NULL;
+ }
+ H5MM_free(chunk_entry->async_info.receive_buffer_array[i]);
+ } /* end for */
+
+ /* Filter the chunk */
+ if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, &filter_mask,
+ io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb,
+ (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0)
+ HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed")
+
+#if H5_SIZEOF_SIZE_T > 4
+ /* Check for the chunk expanding too much to encode in a 32-bit value */
+ if (chunk_entry->chunk_states.new_chunk.length > ((size_t) 0xffffffff))
+ HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length")
+#endif
+
+ break;
+ default:
+ HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid I/O operation")
+ } /* end switch */
+
+done:
+ if (chunk_entry->async_info.receive_buffer_array)
+ H5MM_free(chunk_entry->async_info.receive_buffer_array);
+ if (chunk_entry->async_info.receive_requests_array)
+ H5MM_free(chunk_entry->async_info.receive_requests_array);
+ if (mod_data)
+ H5MM_free(mod_data);
+ if (tmp_gath_buf)
+ H5MM_free(tmp_gath_buf);
+ if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator")
+ if (mem_iter)
+ H5MM_free(mem_iter);
+ if (dataspace)
+ if (H5S_close(dataspace) < 0)
+ HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace")
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__filtered_collective_chunk_entry_io() */
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index a6857b9..097fab7 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -617,6 +617,9 @@ H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info,
H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf,
const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
const H5D_dxpl_cache_t *dxpl_cache, void *_buf);
+H5_DLL size_t H5D__gather_mem(const void *_buf,
+ const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
+ const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/);
H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info,
const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
@@ -666,6 +669,8 @@ H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id,
const hsize_t *scaled, H5D_chunk_ud_t *udata);
H5_DLL herr_t H5D__chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes);
H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]);
+H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk,
+ H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[]);
H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id,
hsize_t old_dim[]);
H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id,
@@ -768,8 +773,7 @@ H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info,
* memory and the file */
H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info,
const H5S_t *file_space, const H5S_t *mem_space,
- const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm,
- H5P_genplist_t *dx_plist);
+ const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist);
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c
index 4625c7a..0ae69ee 100644
--- a/src/H5Dscatgath.c
+++ b/src/H5Dscatgath.c
@@ -47,9 +47,6 @@ static herr_t H5D__scatter_file(const H5D_io_info_t *io_info,
static size_t H5D__gather_file(const H5D_io_info_t *io_info,
const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts,
void *buf);
-static size_t H5D__gather_mem(const void *_buf,
- const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
- const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/);
static herr_t H5D__compound_opt_read(size_t nelmts, const H5S_t *mem_space,
H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache,
const H5D_type_info_t *type_info, void *user_buf/*out*/);
@@ -303,6 +300,7 @@ H5D__scatter_mem (const void *_tscat_buf, const H5S_t *space,
HDassert(space);
HDassert(iter);
HDassert(nelmts > 0);
+ HDassert(dxpl_cache);
HDassert(buf);
/* Allocate the vector I/O arrays */
@@ -364,7 +362,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-static size_t
+size_t
H5D__gather_mem(const void *_buf, const H5S_t *space,
H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache,
void *_tgath_buf/*out*/)
@@ -387,6 +385,7 @@ H5D__gather_mem(const void *_buf, const H5S_t *space,
HDassert(space);
HDassert(iter);
HDassert(nelmts > 0);
+ HDassert(dxpl_cache);
HDassert(tgath_buf);
/* Allocate the vector I/O arrays */
diff --git a/src/H5F.c b/src/H5F.c
index 78fce2a..39eca13 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -443,8 +443,6 @@ done:
hid_t
H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
{
- hbool_t ci_load = FALSE; /* whether MDC ci load requested */
- hbool_t ci_write = FALSE; /* whether MDC CI write requested */
H5F_t *new_file = NULL; /*file struct for new file */
hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */
hid_t ret_value; /*return value */
@@ -490,12 +488,6 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
if(NULL == (new_file = H5F_open(filename, flags, fcpl_id, fapl_id, dxpl_id)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file")
- /* Check to see if both SWMR and cache image are requested. Fail if so */
- if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status")
- if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
- HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image")
-
/* Get an atom for the file */
if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file")
@@ -554,8 +546,6 @@ done:
hid_t
H5Fopen(const char *filename, unsigned flags, hid_t fapl_id)
{
- hbool_t ci_load = FALSE; /* whether MDC ci load requested */
- hbool_t ci_write = FALSE; /* whether MDC CI write requested */
H5F_t *new_file = NULL; /*file struct for new file */
hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */
hid_t ret_value; /*return value */
@@ -585,12 +575,6 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id)
if(NULL == (new_file = H5F_open(filename, flags, H5P_FILE_CREATE_DEFAULT, fapl_id, dxpl_id)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to open file")
- /* Check to see if both SWMR and cache image are requested. Fail if so */
- if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status")
- if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
- HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image")
-
/* Get an atom for the file */
if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle")
@@ -1633,7 +1617,7 @@ H5Fstart_swmr_write(hid_t file_id)
if(file->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "file superblock version should be at least 3")
- HDassert(file->shared->latest_flags == H5F_LATEST_ALL_FLAGS);
+ HDassert((file->shared->latest_flags | H5F_LATEST_LAYOUT_MSG) > 0);
/* Should not be marked for SWMR writing mode already */
if(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)
@@ -1647,6 +1631,10 @@ H5Fstart_swmr_write(hid_t file_id)
if(ci_load || ci_write )
HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image")
+ /* Flush the superblock extension */
+ if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, H5AC_ind_read_dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock extension")
+
/* Flush data buffers */
if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index b980b7e..2ab04dc 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -598,7 +598,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
{
int o_flags;
H5FD_core_t *file = NULL;
- H5FD_core_fapl_t *fa = NULL;
+ const H5FD_core_fapl_t *fa = NULL;
H5P_genplist_t *plist; /* Property list pointer */
#ifdef H5_HAVE_WIN32_API
struct _BY_HANDLE_FILE_INFORMATION fileinfo;
@@ -620,7 +620,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
HDassert(H5P_DEFAULT != fapl_id);
if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
- if(NULL == (fa = (H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
+ if(NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info")
/* Build the open flags */
@@ -638,7 +638,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
((file_image_info.buffer == NULL) && (file_image_info.size == 0)));
HDmemset(&sb, 0, sizeof(sb));
if((file_image_info.buffer != NULL) && !(H5F_ACC_CREAT & flags)) {
- if(HDopen(name, o_flags, 0666) >= 0)
+ if(HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW) >= 0)
HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file already exists")
/* If backing store is requested, create and stat the file
@@ -646,7 +646,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
* technically an open.
*/
if(fa->backing_store) {
- if((fd = HDopen(name, o_flags | O_CREAT, 0666)) < 0)
+ if((fd = HDopen(name, o_flags | O_CREAT, H5_POSIX_CREATE_MODE_RW)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create file")
if(HDfstat(fd, &sb) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file")
@@ -656,7 +656,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
* store is off is when the backing_store flag is off and H5F_ACC_CREAT is
* on. */
else if(fa->backing_store || !(H5F_ACC_CREAT & flags)) {
- if((fd = HDopen(name, o_flags, 0666)) < 0)
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file")
if(HDfstat(fd, &sb) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file")
diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c
index 7a4c99c..811ea8e 100644
--- a/src/H5FDdirect.c
+++ b/src/H5FDdirect.c
@@ -487,7 +487,7 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd
o_flags |= O_DIRECT;
/* Open the file */
- if ((fd=HDopen(name, o_flags, 0666))<0)
+ if ((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW))<0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file")
if (HDfstat(fd, &sb)<0)
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 7c6bbd4..7c2297a 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -520,7 +520,7 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
HDgettimeofday(&timeval_start, NULL);
#endif /* H5_HAVE_GETTIMEOFDAY */
/* Open the file */
- if((fd = HDopen(name, o_flags, 0666)) < 0) {
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) {
int myerrno = errno;
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags);
diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c
index 16be455..98e1b1a 100644
--- a/src/H5FDmpi.c
+++ b/src/H5FDmpi.c
@@ -315,6 +315,11 @@ H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, MPI_Comm *comm_new, MPI_Inf
info_dup = info;
}
+ /* Set MPI_ERRORS_RETURN on comm_dup so that MPI failures are not fatal,
+ and return codes can be checked and handled. May 23, 2017 FTW */
+ if (MPI_SUCCESS != (mpi_code = MPI_Comm_set_errhandler(comm_dup, MPI_ERRORS_RETURN)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Errhandler_set failed", mpi_code)
+
/* copy them to the return arguments */
*comm_new = comm_dup;
*info_new = info_dup;
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index b6385fb..32aff0f 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -341,7 +341,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
o_flags |= O_EXCL;
/* Open the file */
- if((fd = HDopen(name, o_flags, 0666)) < 0) {
+ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) {
int myerrno = errno;
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags);
} /* end if */
diff --git a/src/H5FSint.c b/src/H5FSint.c
index 1a41172..4297291 100644
--- a/src/H5FSint.c
+++ b/src/H5FSint.c
@@ -77,6 +77,32 @@
/*******************/
+/*-------------------------------------------------------------------------
+ * Function: H5FS_init
+ *
+ * Purpose: Initialize the interface in case it is unable to initialize
+ * itself soon enough.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Saturday, March 4, 2000
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FS_init(void)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+ /* FUNC_ENTER() does all the work */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FS_init() */
+
/*-------------------------------------------------------------------------
* Function: H5FS__create_flush_depend
diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h
index c0467a6..247d75c 100644
--- a/src/H5FSprivate.h
+++ b/src/H5FSprivate.h
@@ -175,6 +175,9 @@ H5FL_SEQ_EXTERN(H5FS_section_class_t);
/* Library-private Function Prototypes */
/***************************************/
+/* Package initialization routine */
+H5_DLL herr_t H5FS_init(void);
+
/* Free space manager routines */
H5_DLL H5FS_t *H5FS_create(H5F_t *f, hid_t dxpl_id, haddr_t *fs_addr,
const H5FS_create_t *fs_create, uint16_t nclasses,
diff --git a/src/H5Fint.c b/src/H5Fint.c
index e52d539..7371c46 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -678,8 +678,10 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
if(H5P_get(plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'latest format' flag")
/* For latest format or SWMR_WRITE, activate all latest version support */
- if(latest_format || (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE))
+ if(latest_format)
f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
+ else if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
+ f->shared->latest_flags |= H5F_LATEST_LAYOUT_MSG;
if(H5P_get(plist, H5F_ACS_USE_MDC_LOGGING_NAME, &(f->shared->use_mdc_logging)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'use mdc logging' flag")
if(H5P_get(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, &(f->shared->start_mdc_log_on_access)) < 0)
@@ -1184,6 +1186,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
H5F_t *ret_value = NULL; /*actual return value */
char *lock_env_var = NULL;/*env var pointer */
hbool_t use_file_locking; /*read from env var */
+ hbool_t ci_load = FALSE; /* whether MDC ci load requested */
+ hbool_t ci_write = FALSE; /* whether MDC CI write requested */
FUNC_ENTER_NOAPI(NULL)
@@ -1317,6 +1321,12 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
set_flag = TRUE;
} /* end else */
+ /* Check to see if both SWMR and cache image are requested. Fail if so */
+ if(H5C_cache_image_status(file, &ci_load, &ci_write) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get MDC cache image status")
+ if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)))
+ HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, NULL, "can't have both SWMR and cache image")
+
/* Retain the name the file was opened with */
file->open_name = H5MM_xstrdup(name);
@@ -1456,11 +1466,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)
file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS;
- /* Flush the superblock */
+ /* Flush the superblock & superblock extension */
if(H5F_super_dirty(file) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty")
if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock")
+ if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, meta_dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock extension")
/* Remove the file lock for SWMR_WRITE */
if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) {
@@ -1493,6 +1505,7 @@ done:
if((NULL == ret_value) && file)
if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file")
+
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_open() */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 7a5c126..d702506 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -309,16 +309,16 @@ struct H5F_file_t {
H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */
/* File space allocation information */
- H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */
+ H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */
hsize_t fs_threshold; /* Free space section threshold */
- hbool_t fs_persist; /* Free-space persist or not */
+ hbool_t fs_persist; /* Free-space persist or not */
hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */
haddr_t tmp_addr; /* Next address to use for temp. space in the file */
- hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */
+ hbool_t point_of_no_return; /* Flag to indicate that we can't go back and delete a freespace header when it's used up */
H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */
- haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
- H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
+ haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
+ H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
hbool_t first_alloc_dealloc; /* TRUE iff free space managers */
/* are persistant and have not */
/* been used accessed for either */
@@ -333,25 +333,25 @@ struct H5F_file_t {
/* HADDR_UNDEF if no cache image. */
/* Free-space aggregation info */
- unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */
- H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
- H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */
- H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */
+ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */
+ H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
+ H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */
+ H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */
/* Paged aggregation info */
- hsize_t fs_page_size; /* File space page size */
- size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */
+ hsize_t fs_page_size; /* File space page size */
+ size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */
/* Metadata accumulator information */
- H5F_meta_accum_t accum; /* Metadata accumulator info */
+ H5F_meta_accum_t accum; /* Metadata accumulator info */
/* Metadata retry info */
- unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */
- unsigned retries_nbins; /* # of bins for each retries[] */
+ unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */
+ unsigned retries_nbins; /* # of bins for each retries[] */
uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */
/* Object flush info */
- H5F_object_flush_t object_flush; /* Information for object flush callback */
+ H5F_object_flush_t object_flush; /* Information for object flush callback */
};
/*
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 7c70a64..0c6f9cd 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -21,15 +21,15 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
+#include "H5private.h" /* Generic Functions */
#include "H5ACprivate.h" /* Metadata cache */
-#include "H5Eprivate.h" /* Error handling */
+#include "H5Eprivate.h" /* Error handling */
#include "H5Fpkg.h" /* File access */
-#include "H5FDprivate.h" /* File drivers */
-#include "H5Iprivate.h" /* IDs */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5Iprivate.h" /* IDs */
#include "H5MFprivate.h" /* File memory management */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
#include "H5SMprivate.h" /* Shared Object Header Messages */
@@ -158,7 +158,7 @@ H5F_super_ext_open(H5F_t *f, haddr_t ext_addr, H5O_loc_t *ext_ptr)
/* Open the superblock extension object header */
if(H5O_open(ext_ptr) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension")
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -224,12 +224,12 @@ done:
/*-------------------------------------------------------------------------
* Function: H5F__update_super_ext_driver_msg
*
- * Purpose: Update the superblock extension file driver info message if
- * we are using a V 2 superblock. Observe that the function
- * is a NO-OP if the file driver info message does not exist.
+ * Purpose: Update the superblock extension file driver info message if
+ * we are using a V 2 superblock. Observe that the function
+ * is a NO-OP if the file driver info message does not exist.
* This is necessary, as the function is called whenever the
- * EOA is updated, and were it to create the file driver info
- * message, it would find itself in an infinite recursion.
+ * EOA is updated, and were it to create the file driver info
+ * message, it would find itself in an infinite recursion.
*
* Return: Success: SUCCEED
* Failure: FAIL
@@ -267,7 +267,7 @@ H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id)
/* Check for driver info */
H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
- /* Nothing to do unless there is both driver info and
+ /* Nothing to do unless there is both driver info and
* the driver info superblock extension message has
* already been created.
*/
@@ -330,9 +330,13 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */
haddr_t super_addr; /* Absolute address of superblock */
haddr_t eof; /* End of file address */
- unsigned rw_flags; /* Read/write permissions for file */
- hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
+ unsigned rw_flags; /* Read/write permissions for file */
+ hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
herr_t ret_value = SUCCEED; /* Return value */
+#ifdef H5_HAVE_PARALLEL
+ int mpi_rank = 0, mpi_size = 1;
+ int mpi_result;
+#endif /* H5_HAVE_PARALLEL */
FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL)
@@ -354,8 +358,51 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
/* Find the superblock */
- if(H5FD_locate_signature(&fdio_info, &super_addr) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+#ifdef H5_HAVE_PARALLEL
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
+
+ if((mpi_rank = H5F_mpi_get_rank(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Can't get MPI rank")
+
+ if((mpi_size = H5F_mpi_get_size(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size")
+ }
+
+ /* If we are an MPI application with at least two processes, the
+ * following superblock signature location optimization is applicable.
+ *
+ * Note:: For parallel applications which don't setup for using the
+ * HDF5 MPIO driver, we will arrive here with mpi_size == 1.
+ * This occurs because of the variable initialization (above) and the
+ * fact that we have skipped actually calling MPI functions to determine
+ * our MPI rank and size.
+ */
+ if ( mpi_size > 1 ) {
+ MPI_Comm this_comm = MPI_COMM_NULL;
+
+ if ( mpi_rank == 0 ) {
+ if (H5FD_locate_signature(&fdio_info, &super_addr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+ }
+ HDassert(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI));
+
+ if ( MPI_COMM_NULL == (this_comm = H5F_mpi_get_comm(f)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator")
+
+ if ( MPI_SUCCESS !=
+ (mpi_result = MPI_Bcast(&super_addr,sizeof(super_addr), MPI_BYTE, 0, this_comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result)
+ }
+ else {
+ /* Locate the signature as per per the serial library */
+#endif /* H5_HAVE_PARALLEL */
+
+ if (H5FD_locate_signature(&fdio_info, &super_addr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+
+#ifdef H5_HAVE_PARALLEL
+ }
+#endif /* H5_HAVE_PARALLEL */
if(HADDR_UNDEF == super_addr)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found")
@@ -406,12 +453,12 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock")
if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
- if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
- HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE")
+ if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE")
/* Enable all latest version support when file has v3 superblock */
if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3)
- f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
+ f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
/* Pin the superblock in the cache */
if(H5AC_pin_protected_entry(sblock) < 0)
@@ -511,15 +558,15 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
* been flushed to disk by the SWMR writer process.
*/
if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) {
- /*
- * When the file is opened for SWMR read access, skip the check if:
- * --the file is already marked for SWMR writing and
- * --the file has version 3 superblock for SWMR support
- */
- if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) &&
+ /*
+ * When the file is opened for SWMR read access, skip the check if:
+ * --the file is already marked for SWMR writing and
+ * --the file has version 3 superblock for SWMR support
+ */
+ if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) &&
(sblock->status_flags & H5F_SUPER_WRITE_ACCESS) &&
sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3)
- skip_eof_check = TRUE;
+ skip_eof_check = TRUE;
} /* end if */
if(!skip_eof_check && initial_read) {
if(HADDR_UNDEF == (eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT)))
@@ -593,7 +640,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
H5O_loc_t ext_loc; /* "Object location" for superblock extension */
H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */
H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */
- size_t u; /* Local index variable */
+ size_t u; /* Local index variable */
htri_t status; /* Status for message existing */
/* Sanity check - superblock extension should only be defined for
@@ -614,7 +661,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
} /* end if */
/* Open the superblock extension */
- if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
+ if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
/* Check for the extension having a 'driver info' message */
@@ -637,8 +684,8 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
/* Reset driver info message */
H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo);
- HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
- f->shared->drvinfo_sb_msg_exists = TRUE;
+ HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
+ f->shared->drvinfo_sb_msg_exists = TRUE;
} /* end else */
} /* end if */
@@ -764,37 +811,37 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial
} /* end if not marked "unknown" */
} /* end if */
- /* Check for the extension having a 'metadata cache image' message */
+ /* Check for the extension having a 'metadata cache image' message */
if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
- hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0);
- H5O_mdci_t mdci_msg;
+ hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0);
+ H5O_mdci_t mdci_msg;
- /* if the metadata cache image superblock extension message exists,
+ /* if the metadata cache image superblock extension message exists,
* read its contents and pass the data on to the metadata cache.
* Given this data, the cache will load and decode the metadata
- * cache image block, decoded it and load its contents into the
- * the cache on the test protect call.
+ * cache image block, decoded it and load its contents into the
+ * the cache on the test protect call.
*
* Further, if the file is opened R/W, the metadata cache will
- * delete the metadata cache image superblock extension and free
- * the cache image block. Don't do this now as f->shared
- * is not fully setup, which complicates matters.
+ * delete the metadata cache image superblock extension and free
+ * the cache image block. Don't do this now as f->shared
+ * is not fully setup, which complicates matters.
*/
/* Retrieve the 'metadata cache image message' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id))
+ if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message")
/* Indicate to the cache that there's an image to load on first protect call */
if(H5AC_load_cache_image_on_next_protect(f, mdci_msg.addr, mdci_msg.size, rw) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed");
+ HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed");
} /* end if */
/* Close superblock extension */
if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
} /* end if */
/* Update the driver info if VFD indicated to do so */
@@ -974,6 +1021,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Bump superblock version if latest superblock version support is enabled */
if(H5F_USE_LATEST_FLAGS(f, H5F_LATEST_SUPERBLOCK))
super_vers = HDF5_SUPERBLOCK_VERSION_LATEST;
+ /* Bump superblock version to use version 3 superblock for SWMR writing */
+ else if((H5F_INTENT(f) & H5F_ACC_SWMR_WRITE))
+ super_vers = HDF5_SUPERBLOCK_VERSION_3;
/* Bump superblock version to create superblock extension for SOHM info */
else if(f->shared->sohm_nindexes > 0)
super_vers = HDF5_SUPERBLOCK_VERSION_2;
diff --git a/src/H5I.c b/src/H5I.c
index 42edf58..b8e47a2 100644
--- a/src/H5I.c
+++ b/src/H5I.c
@@ -451,9 +451,9 @@ H5I_nmembers(H5I_type_t type)
FUNC_ENTER_NOAPI(FAIL)
if(type <= H5I_BADID || type >= H5I_next_type)
- HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number")
if(NULL == (type_ptr = H5I_id_type_list_g[type]) || type_ptr->init_count <= 0)
- HGOTO_DONE(0);
+ HGOTO_DONE(0);
/* Set return value */
H5_CHECKED_ASSIGN(ret_value, int64_t, type_ptr->id_count, uint64_t);
diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h
index 7bc3c22..c737bbe 100644
--- a/src/H5Ipublic.h
+++ b/src/H5Ipublic.h
@@ -30,6 +30,8 @@
* When adding types here, add a section to the 'misc19' test in test/tmisc.c
* to verify that the H5I{inc|dec|get}_ref() routines work correctly with it.
*
+ * NOTE: H5I_REFERENCE is not used by the library and has been deprecated
+ * with a tentative removal version of 1.12.0. (DER, July 2017)
*/
typedef enum H5I_type_t {
H5I_UNINIT = (-2), /* uninitialized type */
@@ -40,12 +42,7 @@ typedef enum H5I_type_t {
H5I_DATASPACE, /* type ID for Dataspace objects */
H5I_DATASET, /* type ID for Dataset objects */
H5I_ATTR, /* type ID for Attribute objects */
- H5I_REFERENCE, /* type ID for Reference objects
- ** DEPRECATED**
- H5I_REFERENCE is not used in the library and
- will be removed from this enumeration in a
- future major release of the library.
- */
+ H5I_REFERENCE, /* *DEPRECATED* type ID for Reference objects */
H5I_VFL, /* type ID for virtual file layer */
H5I_GENPROP_CLS, /* type ID for generic property list classes */
H5I_GENPROP_LST, /* type ID for generic property lists */
diff --git a/src/H5MF.c b/src/H5MF.c
index d7af56a..49c7b77 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -115,6 +115,7 @@ hbool_t H5_PKG_INIT_VAR = FALSE;
/* Local Variables */
/*******************/
+
/*-------------------------------------------------------------------------
* Function: H5MF_init_merge_flags
diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c
index 1ae902f..0b300ba 100644
--- a/src/H5MFdbg.c
+++ b/src/H5MFdbg.c
@@ -155,8 +155,8 @@ done:
herr_t
H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int indent, int fwidth)
{
- herr_t ret_value = SUCCEED; /* Return value */
H5F_mem_page_t type; /* Memory type for iteration */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
diff --git a/src/H5Oefl.c b/src/H5Oefl.c
index 49c442f..ba7a6ee 100644
--- a/src/H5Oefl.c
+++ b/src/H5Oefl.c
@@ -151,7 +151,7 @@ H5O_efl_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh,
if((s = (const char *)H5HL_offset_into(heap, mesg->slot[u].name_offset)) == NULL)
HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "unable to get external file name")
- if(*s == (char)NULL)
+ if(*s == (char)'\0')
HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "invalid external file name")
mesg->slot[u].name = H5MM_xstrdup (s);
HDassert(mesg->slot[u].name);
diff --git a/src/H5Oflush.c b/src/H5Oflush.c
index 2d93221..9764f56 100644
--- a/src/H5Oflush.c
+++ b/src/H5Oflush.c
@@ -370,40 +370,40 @@ H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id, hbool_
type = H5I_get_type(oid);
switch(type) {
- case(H5I_GROUP):
+ case H5I_GROUP:
/* Re-open the group */
if(NULL == (object = H5G_open(obj_loc, dxpl_id)))
HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group")
break;
- case(H5I_DATATYPE):
+ case H5I_DATATYPE:
/* Re-open the named datatype */
if(NULL == (object = H5T_open(obj_loc, dxpl_id)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTOPENOBJ, FAIL, "unable to open named datatype")
break;
- case(H5I_DATASET):
+ case H5I_DATASET:
/* Re-open the dataset */
if(NULL == (object = H5D_open(obj_loc, H5P_DATASET_ACCESS_DEFAULT, dxpl_id)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset")
- if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */
- if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset")
+ if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */
+ if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset")
break;
- case(H5I_UNINIT):
- case(H5I_BADID):
- case(H5I_FILE):
- case(H5I_DATASPACE):
- case(H5I_ATTR):
- case(H5I_REFERENCE):
- case(H5I_VFL):
- case(H5I_GENPROP_CLS):
- case(H5I_GENPROP_LST):
- case(H5I_ERROR_CLASS):
- case(H5I_ERROR_MSG):
- case(H5I_ERROR_STACK):
- case(H5I_NTYPES):
+ case H5I_UNINIT:
+ case H5I_BADID:
+ case H5I_FILE:
+ case H5I_DATASPACE:
+ case H5I_ATTR:
+ case H5I_REFERENCE:
+ case H5I_VFL:
+ case H5I_GENPROP_CLS:
+ case H5I_GENPROP_LST:
+ case H5I_ERROR_CLASS:
+ case H5I_ERROR_MSG:
+ case H5I_ERROR_STACK:
+ case H5I_NTYPES:
default:
HGOTO_ERROR(H5E_ARGS, H5E_CANTRELEASE, FAIL, "not a valid file object ID (dataset, group, or datatype)")
break;
diff --git a/src/H5Otest.c b/src/H5Otest.c
index f0deade..68462a1 100644
--- a/src/H5Otest.c
+++ b/src/H5Otest.c
@@ -731,7 +731,7 @@ H5O_msg_move_to_new_chunk_test(hid_t oid, unsigned msg_type)
/* Allocate and initialize new chunk in the file, moving the found message */
/* (*new_idx returned from this routine is unused here) */
- if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, 40, oh->nmesgs, &found_msg, &new_idx) < 0)
+ if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, (curr_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh)), oh->nmesgs, &found_msg, &new_idx) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate new object header chunk")
/* Break out of loop, the message was found */
diff --git a/src/H5PB.c b/src/H5PB.c
index 52576f8..63e5e7a 100644
--- a/src/H5PB.c
+++ b/src/H5PB.c
@@ -1506,7 +1506,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry)
HDassert(page_entry);
/* Retrieve the 'eoa' for the file */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type)))
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, (H5FD_mem_t)page_entry->type)))
HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed")
/* If the starting address of the page is larger than
@@ -1525,7 +1525,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry)
fdio_info.meta_dxpl = fio_info->meta_dxpl;
fdio_info.raw_dxpl = fio_info->raw_dxpl;
- if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0)
+ if(H5FD_write(&fdio_info, (H5FD_mem_t)page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0)
HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed")
} /* end if */
diff --git a/src/H5PL.c b/src/H5PL.c
index 65d6c91..be12db5 100644
--- a/src/H5PL.c
+++ b/src/H5PL.c
@@ -22,135 +22,28 @@
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
-#include "H5MMprivate.h" /* Memory management */
#include "H5PLpkg.h" /* Plugin */
-#include "H5Zprivate.h" /* Filter pipeline */
/****************/
/* Local Macros */
/****************/
-#ifdef H5_HAVE_WIN32_API
-#define H5PL_EXPAND_ENV_VAR { \
- long bufCharCount; \
- char *tempbuf; \
- if(NULL == (tempbuf = (char *)H5MM_malloc(H5PL_EXPAND_BUFFER_SIZE))) \
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") \
- if((bufCharCount = ExpandEnvironmentStringsA(dl_path, tempbuf, H5PL_EXPAND_BUFFER_SIZE)) > H5PL_EXPAND_BUFFER_SIZE) { \
- tempbuf = (char *)H5MM_xfree(tempbuf); \
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") \
- } \
- if(bufCharCount == 0) { \
- tempbuf = (char *)H5MM_xfree(tempbuf); \
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") \
- } \
- dl_path = (char *)H5MM_xfree(dl_path); \
- dl_path = tempbuf; \
- }
-#else
-#define H5PL_EXPAND_ENV_VAR
-#endif /* H5_HAVE_WIN32_API */
-
-/****************************/
-/* Macros for supporting
- * both Windows and Unix */
-/****************************/
-/* Windows support
- *
- * SPECIAL WINDOWS NOTE
- *
- * Some of the Win32 API functions expand to fooA or fooW depending on
- * whether UNICODE or _UNICODE are defined. You MUST explicitly use
- * the A version of the functions to force char * behavior until we
- * work out a scheme for proper Windows Unicode support.
- *
- * If you do not do this, people will be unable to incorporate our
- * source code into their own CMake builds if they define UNICODE.
- */
-#ifdef H5_HAVE_WIN32_API
-
-#define H5PL_PATH_SEPARATOR ";"
-
-/* Handle for dynamic library */
-#define H5PL_HANDLE HINSTANCE
-
-/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
-#define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)
-
-/* Get the address of a symbol in dynamic library */
-#define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N)
-
-/* Close dynamic library */
-#define H5PL_CLOSE_LIB(H) FreeLibrary(H)
-
-/* Clear error - nothing to do */
-#define H5PL_CLR_ERROR
-
-/* maximum size for expanding env vars */
-#define H5PL_EXPAND_BUFFER_SIZE 32767
-
-typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void);
-
-/* Unix support */
-#else /* H5_HAVE_WIN32_API */
-
-#define H5PL_PATH_SEPARATOR ":"
-
-/* Handle for dynamic library */
-#define H5PL_HANDLE void *
-
-/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
-#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY)
-
-/* Get the address of a symbol in dynamic library */
-#define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N)
-
-/* Close dynamic library */
-#define H5PL_CLOSE_LIB(H) dlclose(H)
-
-/* Clear error */
-#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror())
-
-typedef const void *(*H5PL_get_plugin_info_t)(void);
-#endif /* H5_HAVE_WIN32_API */
-
-/* Whether to preload pathnames for plugin libraries */
-#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR
-
-/* Special symbol to indicate no plugin loading */
-#define H5PL_NO_PLUGIN "::"
/******************/
/* Local Typedefs */
/******************/
-/* Type for the list of info for opened plugin libraries */
-typedef struct H5PL_table_t {
- H5PL_type_t pl_type; /* plugin type */
- int pl_id; /* ID for the plugin */
- H5PL_HANDLE handle; /* plugin handle */
-} H5PL_table_t;
-
/********************/
/* Local Prototypes */
/********************/
-static herr_t H5PL__init_path_table(void);
-static htri_t H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info);
-static htri_t H5PL__open(H5PL_type_t pl_type, char *libname, int plugin_id, const void **pl_info);
-static htri_t H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info);
-static herr_t H5PL__close(H5PL_HANDLE handle);
-
/*********************/
/* Package Variables */
/*********************/
-/* Package initialization variable */
-hbool_t H5_PKG_INIT_VAR = FALSE;
-
/*****************************/
/* Library Private Variables */
@@ -161,145 +54,42 @@ hbool_t H5_PKG_INIT_VAR = FALSE;
/* Local Variables */
/*******************/
-/* Table for opened plugin libraries */
-static size_t H5PL_table_alloc_g = 0;
-static size_t H5PL_table_used_g = 0;
-static H5PL_table_t *H5PL_table_g = NULL;
-
-/* Table of location paths for plugin libraries */
-static char *H5PL_path_table_g[H5PL_MAX_PATH_NUM];
-static size_t H5PL_num_paths_g = 0;
-static hbool_t H5PL_path_found_g = FALSE;
-
-/* Enable all plugin libraries */
-static unsigned int H5PL_plugin_g = H5PL_ALL_PLUGIN;
-
-
-
-/*--------------------------------------------------------------------------
-NAME
- H5PL__init_package -- Initialize interface-specific information
-USAGE
- herr_t H5PL__init_package()
-RETURNS
- Non-negative on success/Negative on failure
-DESCRIPTION
- Initializes any interface-specific data or routines.
-
---------------------------------------------------------------------------*/
-herr_t
-H5PL__init_package(void)
-{
- char *preload_path;
-
- FUNC_ENTER_PACKAGE_NOERR
-
- /* Retrieve pathnames from HDF5_PLUGIN_PRELOAD if the user sets it
- * to tell the library to load plugin libraries without search.
- */
- if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD")))
- /* Special symbol "::" means no plugin during data reading. */
- if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN))
- H5PL_plugin_g = 0;
-
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5PL__init_package() */
-
/*-------------------------------------------------------------------------
- * Function: H5PL_term_package
+ * Function: H5PLset_loading_state
*
- * Purpose: Terminate the H5PL interface: release all memory, reset all
- * global variables to initial values. This only happens if all
- * types have been destroyed from other interfaces.
+ * Purpose: Control the loading of dynamic plugin types.
*
- * Return: Success: Positive if any action was taken that might
- * affect some other interface; zero otherwise.
- * Failure: Negative.
+ * The plugin_control_mask parameter is a bitfield that controls
+ * whether certain classes of plugins (e.g.: filters,
+ * VOL drivers) will be loaded by the library.
*
- * Programmer: Raymond Lu
- * 20 February 2013
+ * plugin bit = 0, will prevent the use of that dynamic plugin type.
+ * plugin bit = 1, will allow the use of that dynamic plugin type.
*
- *-------------------------------------------------------------------------
- */
-int
-H5PL_term_package(void)
-{
- int n = 0;
-
- FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- if(H5_PKG_INIT_VAR) {
- size_t u; /* Local index variable */
-
- /* Close opened dynamic libraries */
- if(H5PL_table_g) {
- for(u = 0; u < H5PL_table_used_g; u++)
- H5PL__close((H5PL_table_g[u]).handle);
-
- /* Free the table of dynamic libraries */
- H5PL_table_g = (H5PL_table_t *)H5MM_xfree(H5PL_table_g);
- H5PL_table_used_g = H5PL_table_alloc_g = 0;
-
- n++;
- } /* end if */
-
- /* Free the table of search paths */
- if(H5PL_num_paths_g > 0) {
- for(u = 0; u < H5PL_num_paths_g; u++)
- if(H5PL_path_table_g[u])
- H5PL_path_table_g[u] = (char *)H5MM_xfree(H5PL_path_table_g[u]);
- H5PL_num_paths_g = 0;
- H5PL_path_found_g = FALSE;
-
- n++;
- } /* end if */
-
- /* Mark the interface as uninitialized */
- if(0 == n)
- H5_PKG_INIT_VAR = FALSE;
- } /* end if */
-
- FUNC_LEAVE_NOAPI(n)
-} /* end H5PL_term_package() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PLset_loading_state
- *
- * Purpose: Control the loading of dynamic plugin types.
- *
- * This function will not allow plugin types if the pathname from the HDF5_PLUGIN_PRELOAD
- * environment variable is set to the special "::" string.
+ * A list of pre-defined masks can be found in H5PLpublic.h.
+ * Set the mask to 0 to disable all plugins.
*
- * plugin bit = 0, will prevent the use of that dynamic plugin type.
- * plugin bit = 1, will allow the use of that dynamic plugin type.
+ * This function will not allow plugin types if the pathname
+ * from the HDF5_PLUGIN_PRELOAD environment variable is set to
+ * the special "::" string.
*
- * H5PL_TYPE_FILTER changes just dynamic filters
- * A H5PL_ALL_PLUGIN will enable all dynamic plugin types
- * A zero value will disable all dynamic plugin types
- *
- * Return: Non-negative or success
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLset_loading_state(unsigned int plugin_type)
+H5PLset_loading_state(unsigned int plugin_control_mask)
{
- char *preload_path;
- herr_t ret_value = SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "Iu", plugin_type);
-
- /* change the bit value of the requested plugin type(s) */
- H5PL_plugin_g = plugin_type;
+ H5TRACE1("e", "Iu", plugin_control_mask);
- /* check if special ENV variable is set and disable all plugin types */
- if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD")))
- /* Special symbol "::" means no plugin during data reading. */
- if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN))
- H5PL_plugin_g = 0;
+ /* Set the plugin control mask */
+ if(H5PL__set_plugin_control_mask(plugin_control_mask) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTSET, FAIL, "error setting plugin control mask")
done:
FUNC_LEAVE_API(ret_value)
@@ -307,27 +97,35 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5PLget_loading_state
+ * Function: H5PLget_loading_state
*
- * Purpose: Query state of the loading of dynamic plugin types.
+ * Purpose: Get the bitmask that controls whether certain classes
+ * of plugins (e.g.: filters, VOL drivers) will be loaded
+ * by the library.
*
- * This function will return the state of the global flag.
+ * Zero if all plugin types are disabled
+ * Negative if all plugin types are enabled
+ * Positive if one or more of the plugin types are enabled
*
- * Return: Zero if all plugin types are disabled, negative if all
- * plugin types are enabled, positive if one or more of the plugin types are enabled.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLget_loading_state(unsigned int *plugin_type)
+H5PLget_loading_state(unsigned int *plugin_control_mask)
{
- herr_t ret_value = SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*Iu", plugin_type);
+ H5TRACE1("e", "*Iu", plugin_control_mask);
+
+ if (NULL == plugin_control_mask)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_control_mask parameter cannot be NULL")
- if(plugin_type)
- *plugin_type = H5PL_plugin_g;
+ /* Set the plugin control mask */
+ if(H5PL__get_plugin_control_mask(plugin_control_mask) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error getting plugin control mask")
done:
FUNC_LEAVE_API(ret_value)
@@ -335,674 +133,282 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5PL_load
- *
- * Purpose: Given the plugin type and identifier, this function searches
- * and/or loads a dynamic plugin library first among the already
- * opened libraries then in the designated location paths.
- *
- * Return: Non-NULL on success/NULL on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-const void *
-H5PL_load(H5PL_type_t type, int id)
-{
- htri_t found; /* Whether the plugin was found */
- const void *plugin_info = NULL;
- const void *ret_value = NULL;
-
- FUNC_ENTER_NOAPI(NULL)
-
- switch(type) {
- case H5PL_TYPE_FILTER:
- if((H5PL_plugin_g & H5PL_FILTER_PLUGIN) == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id)
- break;
-
- case H5PL_TYPE_ERROR:
- case H5PL_TYPE_NONE:
- default:
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id)
- } /* end switch */
-
- /* Initialize the location paths for dynamic libraries, if they aren't
- * already set up.
- */
- if(FALSE == H5PL_path_found_g)
- if(H5PL__init_path_table() < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, NULL, "can't initialize search path table")
-
- /* Search in the table of already loaded plugin libraries */
- if((found = H5PL__search_table(type, id, &plugin_info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in table failed")
-
- /* If not found, iterate through the path table to find the right dynamic library */
- if(!found) {
- size_t i; /* Local index variable */
-
- for(i = 0; i < H5PL_num_paths_g; i++) {
- if((found = H5PL__find(type, id, H5PL_path_table_g[i], &plugin_info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in paths failed")
-
- /* Break out if found */
- if(found) {
- HDassert(plugin_info);
- break;
- } /* end if */
- } /* end for */
- } /* end if */
-
- /* Check if we found the plugin */
- if(found)
- ret_value = plugin_info;
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL_load() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PLappend
+ * Function: H5PLappend
*
- * Purpose: Insert a plugin path at the end of the list.
+ * Purpose: Insert a plugin search path at the end of the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLappend(const char *plugin_path)
+H5PLappend(const char *search_path)
{
herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*s", plugin_path);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+ H5TRACE1("e", "*s", search_path);
- H5PL_EXPAND_ENV_VAR
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- H5PL_path_table_g[H5PL_num_paths_g] = dl_path;
- H5PL_num_paths_g++;
+ /* Append the search path to the path table */
+ if (H5PL__append_path(search_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTAPPEND, FAIL, "unable to append search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLappend() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLprepend
+ * Function: H5PLprepend
*
- * Purpose: Insert a plugin path at the beginning of the list.
+ * Purpose: Insert a plugin search path at the beginning of the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLprepend(const char *plugin_path)
+H5PLprepend(const char *search_path)
{
herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
- unsigned int plindex;
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*s", plugin_path);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+ H5TRACE1("e", "*s", search_path);
- H5PL_EXPAND_ENV_VAR
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- for (plindex = (unsigned int)H5PL_num_paths_g; plindex > 0; plindex--)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1];
- H5PL_path_table_g[0] = dl_path;
- H5PL_num_paths_g++;
+ /* Prepend the search path to the path table */
+ if (H5PL__prepend_path(search_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLprepend() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLreplace
+ * Function: H5PLreplace
*
- * Purpose: Replace the path at the specified index.
+ * Purpose: Replace the path at the specified index. The path at the
+ * index must exist.
*
- * Return: Non-negative or success.
+ * Return: Non-negative or success.
*
*-------------------------------------------------------------------------
*/
herr_t
H5PLreplace(const char *plugin_path, unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "*sIu", plugin_path, index);
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- H5PL_EXPAND_ENV_VAR
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
- if(H5PL_path_table_g[index])
- H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]);
- H5PL_path_table_g[index] = dl_path;
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Insert the search path into the path table */
+ if (H5PL__replace_path(search_path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLreplace() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLinsert
+ * Function: H5PLinsert
*
- * Purpose: Insert a plugin path at the specified index, moving other paths after the index.
+ * Purpose: Insert a plugin search path at the specified index, moving
+ * other paths after the index.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLinsert(const char *plugin_path, unsigned int index)
+H5PLinsert(const char *search_path, unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- char *dl_path = NULL;
- unsigned int plindex;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE2("e", "*sIu", plugin_path, index);
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == plugin_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5MM_strdup(plugin_path)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
-
- H5PL_EXPAND_ENV_VAR
-
- for(plindex = (unsigned int)H5PL_num_paths_g; plindex > index; plindex--)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1];
- H5PL_path_table_g[index] = dl_path;
- H5PL_num_paths_g++;
+ H5TRACE2("e", "*sIu", search_path, index);
+
+ /* Check args */
+ if (NULL == search_path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL")
+ if (0 == HDstrlen(search_path))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero")
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if ((0 != num_paths) && (index >= num_paths))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Insert the search path into the path table */
+ if (H5PL__insert_path(search_path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLinsert() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLremove
+ * Function: H5PLremove
*
- * Purpose: Remove the plugin path at the specifed index and compacting the list.
+ * Purpose: Remove the plugin path at the specifed index and compact
+ * the list.
*
- * Return: Non-negative or success.
+ * Return: Success: Non-negative
+ * Failture: Negative
+ *
+ * Return: Non-negative or success.
*
*-------------------------------------------------------------------------
*/
herr_t
H5PLremove(unsigned int index)
{
- herr_t ret_value = SUCCEED; /* Return value */
- unsigned int plindex;
+ unsigned num_paths; /* Current number of stored paths */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE1("e", "Iu", index);
- if(H5PL_num_paths_g == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == H5PL_path_table_g[index])
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index")
- H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]);
-
- H5PL_num_paths_g--;
- for(plindex = index; plindex < (unsigned int)H5PL_num_paths_g; plindex++)
- H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex + 1];
- H5PL_path_table_g[H5PL_num_paths_g] = NULL;
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Delete the search path from the path table */
+ if (H5PL__remove_path(index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "unable to remove search path")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLremove() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLget
+ * Function: H5PLget
+ *
+ * Purpose: Query the plugin path at a specified index.
*
- * Purpose: Query the plugin path at the specified index.
+ * If 'path_buf' is non-NULL then up to 'buf_size' bytes will be written into
+ * that buffer and the length of the path name will be returned.
*
- * Return: Success: The length of path.
+ * If 'path_buf' is NULL, this function will simply return the number of
+ * characters required to store the path name, ignoring 'path_buf' and
+ * 'buf_size'
*
- * If `pathname' is non-NULL then write up to `size' bytes into that
- * buffer and always return the length of the pathname.
- * Otherwise `size' is ignored and the function does not store the pathname,
- * just returning the number of characters required to store the pathname.
- * If an error occurs then the buffer pointed to by `pathname' (NULL or non-NULL)
- * is unchanged and the function returns a negative value.
- * If a zero is returned for the name's length, then there is no pathname
- * associated with the index.
+ * If an error occurs then the buffer pointed to by 'path_buf'
+ * (NULL or non-NULL) will be unchanged and the function will return a
+ * negative value.
+ *
+ * If a zero is returned for the name's length, then there is no path name
+ * associated with the index and the 'path_buf' buffer will be unchanged.
+ *
+ * Return: Success: The length of path
+ * Failure: A negative value
*
*-------------------------------------------------------------------------
*/
ssize_t
-H5PLget(unsigned int index, char *pathname/*out*/, size_t size)
+H5PLget(unsigned int index, char *path_buf, size_t buf_size)
{
- ssize_t ret_value = 0; /* Return value */
- size_t len = 0; /* Length of pathname */
- char *dl_path = NULL;
+ unsigned num_paths; /* Current number of stored paths */
+ const char *path = NULL; /* path from table */
+ size_t path_len = 0; /* Length of path */
+ ssize_t ret_value = 0; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE3("Zs", "Iuxz", index, pathname, size);
- if(H5PL_num_paths_g == 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table")
- if(index >= H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table")
- if(NULL == (dl_path = H5PL_path_table_g[index]))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index")
- len = HDstrlen(dl_path);
- if(pathname) {
- HDstrncpy(pathname, dl_path, MIN((size_t)(len + 1), size));
- if((size_t)len >= size)
- pathname[size - 1] = '\0';
+ H5TRACE3("Zs", "Iu*sz", index, path_buf, buf_size);
+
+ /* Check index */
+ num_paths = H5PL__get_num_paths();
+ if (0 == num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty")
+ else if (index >= num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1))
+
+ /* Check if the search table is empty */
+ if (H5PL__get_num_paths() == 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, (-1), "plugin search path table is empty")
+
+ /* Get the path at the specified index and its length */
+ if (NULL == (path = H5PL__get_path(index)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, (-1), "no path stored at that index")
+ path_len = HDstrlen(path);
+
+ /* If the path buffer is not NULL, copy the path to the buffer */
+ if (path_buf) {
+ HDstrncpy(path_buf, path, MIN((size_t)(path_len + 1), buf_size));
+ if ((size_t)path_len >= buf_size)
+ path_buf[buf_size - 1] = '\0';
} /* end if */
/* Set return value */
- ret_value = (ssize_t)len;
+ ret_value = (ssize_t)path_len;
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLget() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5PLsize
+ * Function: H5PLsize
*
- * Purpose: Query the size of the current list of plugin paths.
+ * Purpose: Get the number of stored plugin paths.
+ * XXX: This is a terrible name. Can it be changed?
*
- * Return: Plugin path size
+ * Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
-H5PLsize(unsigned int *listsize)
+H5PLsize(unsigned int *num_paths)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE1("e", "*Iu", listsize);
+ H5TRACE1("e", "*Iu", num_paths);
+
+ /* Check arguments */
+ if (!num_paths)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "num_paths parameter cannot be NULL")
- *listsize = (unsigned int)H5PL_num_paths_g;
+ /* Get the number of stored plugin paths */
+ *num_paths = H5PL__get_num_paths();
done:
FUNC_LEAVE_API(ret_value)
} /* end H5PLsize() */
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__init_path_table
- *
- * Purpose: Initialize the path table.
- *
- * Return: Non-negative on success/Negative on failure
- *
- * Programmer: Quincey Koziol
- * 18 March 2013
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5PL__init_path_table(void)
-{
- char *dl_path = NULL;
- char *origin_dl_path;
- char *dir;
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_STATIC
-
- /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it
- * or from the default paths if it isn't set.
- */
- origin_dl_path = HDgetenv("HDF5_PLUGIN_PATH");
- if(NULL == origin_dl_path)
- dl_path = H5MM_strdup(H5PL_DEFAULT_PATH);
- else
- dl_path = H5MM_strdup(origin_dl_path);
- if(NULL == dl_path)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
-
- H5PL_EXPAND_ENV_VAR
-
- /* Put paths in the path table. They are separated by ":" */
- dir = HDstrtok(dl_path, H5PL_PATH_SEPARATOR);
- while(dir) {
- /* Check for too many directories in path */
- if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM)
- HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table")
- if(NULL == (H5PL_path_table_g[H5PL_num_paths_g] = H5MM_strdup(dir)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- H5PL_num_paths_g++;
- dir = HDstrtok(NULL, H5PL_PATH_SEPARATOR);
- } /* end while */
-
- H5PL_path_found_g = TRUE;
-
-done:
- if(dl_path)
- dl_path = (char *)H5MM_xfree(dl_path);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__init_path_table() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__find
- *
- * Purpose: Given a path, this function opens the directory and envokes
- * another function to go through all files to find the right
- * plugin library. Two function definitions are for Unix and
- * Windows.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-#ifndef H5_HAVE_WIN32_API
-static htri_t
-H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info)
-{
- char *pathname = NULL;
- DIR *dirp = NULL;
- struct dirent *dp;
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Open the directory */
- if(!(dirp = HDopendir(dir)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir)
-
- /* Iterates through all entries in the directory to find the right plugin library */
- while(NULL != (dp = HDreaddir(dirp))) {
- /* The library we are looking for should be called libxxx.so... on Unix
- * or libxxx.xxx.dylib on Mac.
- */
-#ifndef __CYGWIN__
- if(!HDstrncmp(dp->d_name, "lib", (size_t)3) &&
- (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) {
-#else
- if(!HDstrncmp(dp->d_name, "cyg", (size_t)3) &&
- HDstrstr(dp->d_name, ".dll") ) {
-
-#endif
- h5_stat_t my_stat;
- size_t pathname_len;
- htri_t found_in_dir;
-
- /* Allocate & initialize the path name */
- pathname_len = HDstrlen(dir) + HDstrlen(dp->d_name) + 2;
- if(NULL == (pathname = (char *)H5MM_malloc(pathname_len)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- HDsnprintf(pathname, pathname_len, "%s/%s", dir, dp->d_name);
-
- /* Get info for directory entry */
- if(HDstat(pathname, &my_stat) == -1)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file: %s", HDstrerror(errno))
-
- /* If it is a directory, skip it */
- if(S_ISDIR(my_stat.st_mode))
- continue;
-
- /* Attempt to open the dynamic library as a filter library */
- if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
- if(found_in_dir)
- HGOTO_DONE(TRUE) /* Indicate success */
- pathname = (char *)H5MM_xfree(pathname);
- } /* end if */
- } /* end while */
-
-done:
- if(dirp)
- if(HDclosedir(dirp) < 0)
- HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno))
- pathname = (char *)H5MM_xfree(pathname);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__find() */
-#else /* H5_HAVE_WIN32_API */
-static htri_t
-H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info)
-{
- WIN32_FIND_DATAA fdFile;
- HANDLE hFind;
- char *pathname = NULL;
- char service[2048];
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Specify a file mask. *.* = We want everything! */
- sprintf(service, "%s\\*.dll", dir);
- if((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE)
- HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory")
-
- do {
- /* Find first file will always return "."
- * and ".." as the first two directories.
- */
- if(HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) {
- size_t pathname_len;
- htri_t found_in_dir;
-
- /* Allocate & initialize the path name */
- pathname_len = HDstrlen(dir) + HDstrlen(fdFile.cFileName) + 2;
- if(NULL == (pathname = (char *)H5MM_malloc(pathname_len)))
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
- HDsnprintf(pathname, pathname_len, "%s\\%s", dir, fdFile.cFileName);
-
- /* Is the entity a File or Folder? */
- if(fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
- continue;
-
- if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
- if(found_in_dir)
- HGOTO_DONE(TRUE) /* Indicate success */
- pathname = (char *)H5MM_xfree(pathname);
- } /* end if */
- } while(FindNextFileA(hFind, &fdFile)); /* Find the next file. */
-
-done:
- if(hFind)
- FindClose(hFind);
- if(pathname)
- pathname = (char *)H5MM_xfree(pathname);
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__find() */
-#endif /* H5_HAVE_WIN32_API */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__open
- *
- * Purpose: Iterates through all files to find the right plugin library.
- * It loads the dynamic plugin library and keeps it on the list
- * of loaded libraries.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static htri_t
-H5PL__open(H5PL_type_t pl_type, char *libname, int pl_id, const void **pl_info)
-{
- H5PL_HANDLE handle = NULL;
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* There are different reasons why a library can't be open, e.g. wrong architecture.
- * simply continue if we can't open it.
- */
- if(NULL == (handle = H5PL_OPEN_DLIB(libname))) {
- H5PL_CLR_ERROR; /* clear error */
- } /* end if */
- else {
- H5PL_get_plugin_info_t get_plugin_info = NULL;
-
- /* Return a handle for the function H5PLget_plugin_info in the dynamic library.
- * The plugin library is suppose to define this function.
- */
- if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) {
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- } /* end if */
- else {
- const H5Z_class2_t *plugin_info;
-
- /* Invoke H5PLget_plugin_info to verify this is the right library we are looking for.
- * Move on if it isn't.
- */
- if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) {
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info")
- } /* end if */
-
- /* Successfully found plugin library, check if it's the right one */
- if(plugin_info->id == pl_id) {
- /* Expand the table if it is too small */
- if(H5PL_table_used_g >= H5PL_table_alloc_g) {
- size_t n = MAX(H5Z_MAX_NFILTERS, 2 * H5PL_table_alloc_g);
- H5PL_table_t *table = (H5PL_table_t *)H5MM_realloc(H5PL_table_g, n * sizeof(H5PL_table_t));
-
- if(!table)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend dynamic library table")
-
- H5PL_table_g = table;
- H5PL_table_alloc_g = n;
- } /* end if */
-
- (H5PL_table_g[H5PL_table_used_g]).handle = handle;
- (H5PL_table_g[H5PL_table_used_g]).pl_type = pl_type;
- (H5PL_table_g[H5PL_table_used_g]).pl_id = plugin_info->id;
- H5PL_table_used_g++;
-
- /* Set the plugin info to return */
- *pl_info = (const void *)plugin_info;
-
- /* Indicate success */
- ret_value = TRUE;
- } /* end if */
- else
- if(H5PL__close(handle) < 0)
- HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
- } /* end if */
- } /* end else */
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__open() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__search_table
- *
- * Purpose: Search in the list of already opened dynamic libraries
- * to see if the one we are looking for is already opened.
- *
- * Return: TRUE on success,
- * FALSE on not found,
- * Negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static htri_t
-H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info)
-{
- htri_t ret_value = FALSE;
-
- FUNC_ENTER_STATIC
-
- /* Search in the table of already opened dynamic libraries */
- if(H5PL_table_used_g > 0) {
- size_t i;
-
- for(i = 0; i < H5PL_table_used_g; i++) {
- if((plugin_type == (H5PL_table_g[i]).pl_type) && (type_id == (H5PL_table_g[i]).pl_id)) {
- H5PL_get_plugin_info_t get_plugin_info;
- const H5Z_class2_t *plugin_info;
-
- if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_table_g[i]).handle, "H5PLget_plugin_info")))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info")
-
- if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)()))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info")
-
- *info = plugin_info;
- HGOTO_DONE(TRUE)
- } /* end if */
- } /* end for */
- } /* end if */
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5PL__search_table() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5PL__close
- *
- * Purpose: Closes the handle for dynamic library
- *
- * Return: Non-negative on success/Negative on failure
- *
- * Programmer: Raymond Lu
- * 13 February 2013
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5PL__close(H5PL_HANDLE handle)
-{
- FUNC_ENTER_STATIC_NOERR
-
- H5PL_CLOSE_LIB(handle);
-
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5PL__close() */
-
diff --git a/src/H5PLint.c b/src/H5PLint.c
new file mode 100644
index 0000000..c887f86
--- /dev/null
+++ b/src/H5PLint.c
@@ -0,0 +1,384 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Internal routines for managing plugins.
+ *
+ */
+
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+#include "H5Zprivate.h" /* Filter pipeline */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Package initialization variable */
+hbool_t H5_PKG_INIT_VAR = FALSE;
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Bitmask that controls whether classes of plugins
+ * (e.g.: filters, VOL drivers) can be loaded.
+ */
+static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN;
+
+/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD
+ * environment variable was set to H5PL_NO_PLUGIN at
+ * package initialization.
+ */
+static hbool_t H5PL_allow_plugins_g = TRUE;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_plugin_control_mask
+ *
+ * Purpose: Gets the internal plugin control mask value.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__get_plugin_control_mask(unsigned int *mask /*out*/)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Check args - Just assert on package functions */
+ HDassert(mask);
+
+ /* Return the mask */
+ *mask = H5PL_plugin_control_mask_g;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__get_plugin_control_mask() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__set_plugin_control_mask
+ *
+ * Purpose: Sets the internal plugin control mask value.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__set_plugin_control_mask(unsigned int mask)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Only allow setting this if plugins have not been disabled.
+ * XXX: Note that we don't consider this an error, but instead
+ * silently ignore it. We may want to consider this behavior
+ * more carefully.
+ */
+ if (H5PL_allow_plugins_g)
+ H5PL_plugin_control_mask_g = mask;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__set_plugin_control_mask() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__init_package
+ *
+ * Purpose: Initialize any package-specific data and call any init
+ * routines for the package.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__init_package(void)
+{
+ char *env_var = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check the environment variable to determine if the user wants
+ * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in
+ * H5PLpublic.h) means we don't want to load plugins.
+ */
+ if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD")))
+ if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) {
+ H5PL_plugin_control_mask_g = 0;
+ H5PL_allow_plugins_g = FALSE;
+ }
+
+ /* Create the table of previously-loaded plugins */
+ if (H5PL__create_plugin_cache() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache")
+
+ /* Create the table of search paths for dynamic libraries */
+ if (H5PL__create_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__init_package() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL_term_package
+ *
+ * Purpose: Terminate the H5PL interface: release all memory, reset all
+ * global variables to initial values. This only happens if all
+ * types have been destroyed from other interfaces.
+ *
+ * Return: Success: Positive if any action was taken that might
+ * affect some other interface; zero otherwise
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+H5PL_term_package(void)
+{
+ hbool_t already_closed = FALSE;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (H5_PKG_INIT_VAR) {
+
+ /* Close the plugin cache.
+ * We need to bump the return value if we did any real work here.
+ */
+ if (H5PL__close_plugin_cache(&already_closed) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache")
+ if (!already_closed)
+ ret_value++;
+
+ /* Close the search path table and free the paths */
+ if (H5PL__close_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table")
+
+ /* Mark the interface as uninitialized */
+ if (0 == ret_value)
+ H5_PKG_INIT_VAR = FALSE;
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL_term_package() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL_load
+ *
+ * Purpose: Given the plugin type and identifier, this function searches
+ * for and, if found, loads a dynamic plugin library.
+ *
+ * The function searches first in the cached plugins and then
+ * in the paths listed in the path table.
+ *
+ * Return: Success: A pointer to the plugin info
+ * Failure: NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+const void *
+H5PL_load(H5PL_type_t type, int id)
+{
+ H5PL_search_params_t search_params;
+ hbool_t found = FALSE; /* Whether the plugin was found */
+ const void *plugin_info = NULL;
+ const void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI(NULL)
+
+ /* Check if plugins can be loaded for this plugin type */
+ switch (type) {
+ case H5PL_TYPE_FILTER:
+ if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id)
+ break;
+
+ case H5PL_TYPE_ERROR:
+ case H5PL_TYPE_NONE:
+ default:
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id)
+ }
+
+ /* Set up the search parameters */
+ search_params.type = type;
+ search_params.id = id;
+
+ /* Search in the table of already loaded plugin libraries */
+ if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed")
+
+ /* If not found, try iterating through the path table to find an appropriate plugin */
+ if (!found)
+ if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed")
+
+ /* Set the return value we found the plugin */
+ if (found)
+ ret_value = plugin_info;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL_load() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__open
+ *
+ * Purpose: Opens a plugin.
+ *
+ * The success parameter will be set to TRUE and the plugin_info
+ * parameter will be filled in on success. Otherwise, they
+ * will be FALSE and NULL, respectively.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting
+ * object pointers to function pointers, which is undefined in ANSI C.
+ * This is basically unavoidable due to the nature of dlsym() and *is*
+ * defined in POSIX, so it's fine.
+ *
+ * This pragma only needs to surround the assignment of the
+ * get_plugin_info function pointer, but early (4.4.7, at least) gcc
+ * only allows diagnostic pragmas to be toggled outside of functions.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+herr_t
+H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info)
+{
+ H5PL_HANDLE handle = NULL;
+ H5PL_get_plugin_info_t get_plugin_info = NULL;
+ htri_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(success);
+ HDassert(plugin_info);
+
+ /* Initialize out parameters */
+ *success = FALSE;
+ *plugin_info = NULL;
+
+ /* There are different reasons why a library can't be open, e.g. wrong architecture.
+ * If we can't open the library, just return.
+ */
+ if (NULL == (handle = H5PL_OPEN_DLIB(path))) {
+ H5PL_CLR_ERROR; /* clear error */
+ HGOTO_DONE(SUCCEED);
+ }
+
+ /* Return a handle for the function H5PLget_plugin_info in the dynamic library.
+ * The plugin library is suppose to define this function.
+ */
+ if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) {
+
+ const H5Z_class2_t *info;
+
+ /* Get the plugin info */
+ if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)()))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info")
+
+ /* Check if the filter IDs match */
+ if (info->id == id) {
+
+ /* Store the plugin in the cache */
+ if (H5PL__add_plugin(type, id, handle))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache")
+
+ /* Set output parameters */
+ *success = TRUE;
+ *plugin_info = (const void *)info;
+ }
+ }
+
+done:
+ if (!success && handle)
+ if (H5PL__close(handle) < 0)
+ HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__open() */
+#pragma GCC diagnostic pop
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close
+ *
+ * Purpose: Closes the handle for dynamic library
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close(H5PL_HANDLE handle)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ H5PL_CLOSE_LIB(handle);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5PL__close() */
+
diff --git a/src/H5PLpath.c b/src/H5PLpath.c
new file mode 100644
index 0000000..435802a
--- /dev/null
+++ b/src/H5PLpath.c
@@ -0,0 +1,776 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Code to implement a path table which stores plugin search paths.
+ *
+ * The path table is implemented as a dynamic, global array which
+ * will grow as new paths are inserted. The capacity of the path
+ * table never shrinks (though given the low number of paths
+ * expected and the low likelihood of paths being removed, this
+ * seems unlikely to be a problem). Inserts and removals rework
+ * the array so that there are no 'holes' in the in-use part
+ * of the array.
+ *
+ * Note that it's basically up to the user to manage the indexes
+ * when a complicated series of insert, overwrite, and, remove
+ * operations take place.
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+/* Initial capacity of the path table */
+#define H5PL_INITIAL_PATH_CAPACITY 16
+
+/* The amount to add to the capacity when the table is full */
+#define H5PL_PATH_CAPACITY_ADD 16
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static herr_t H5PL__insert_at(const char *path, unsigned int index);
+static herr_t H5PL__make_space_at(unsigned int index);
+static herr_t H5PL__replace_at(const char *path, unsigned int index);
+static herr_t H5PL__expand_path_table(void);
+static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info);
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Stored plugin paths to search */
+static char **H5PL_paths_g = NULL;
+
+/* The number of stored paths */
+static unsigned H5PL_num_paths_g = 0;
+
+/* The capacity of the path table */
+static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__insert_at()
+ *
+ * Purpose: Insert a path at a particular index in the path table.
+ * Does not clobber! Will move existing paths up to make
+ * room. Use H5PL__replace_at(index) if you want to clobber.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__insert_at(const char *path, unsigned int index)
+{
+ char *path_copy = NULL; /* copy of path string (for storing) */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Expand the table if it is full */
+ if (H5PL_num_paths_g == H5PL_path_capacity_g)
+ if (H5PL__expand_path_table() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table")
+
+ /* Copy the path for storage so the caller can dispose of theirs */
+ if (NULL == (path_copy = H5MM_strdup(path)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
+
+#ifdef H5_HAVE_WIN32_API
+ /* Clean up Microsoft Windows environment variables in the path string */
+ if(H5_expand_windows_env_vars(&path_copy))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
+#endif /* H5_HAVE_WIN32_API */
+
+ /* If the table entry is in use, make some space */
+ if (H5PL_paths_g[index])
+ if (H5PL__make_space_at(index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry")
+
+ /* Insert the copy of the search path into the table at the specified index */
+ H5PL_paths_g[index] = path_copy;
+ H5PL_num_paths_g++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__insert_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__make_space_at()
+ *
+ * Purpose: Free up a slot in the path table, moving existing path
+ * entries as necessary.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__make_space_at(unsigned int index)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check args - Just assert on package functions */
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Copy the paths back to make a space */
+ for (u = H5PL_num_paths_g; u > index; u--)
+ H5PL_paths_g[u] = H5PL_paths_g[u-1];
+
+ H5PL_paths_g[index] = NULL;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__make_space_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__replace_at()
+ *
+ * Purpose: Replace a path at a particular index in the path table.
+ * The path in the table must exist and will be freed by this
+ * function.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__replace_at(const char *path, unsigned int index)
+{
+ char *path_copy = NULL; /* copy of path string (for storing) */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Check that the table entry is in use */
+ if (!H5PL_paths_g[index])
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index)
+
+ /* Copy the path for storage so the caller can dispose of theirs */
+ if (NULL == (path_copy = H5MM_strdup(path)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
+
+#ifdef H5_HAVE_WIN32_API
+ /* Clean up Microsoft Windows environment variables in the path string */
+ if (H5_expand_windows_env_vars(&path_copy))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
+#endif /* H5_HAVE_WIN32_API */
+
+ /* Free the existing path entry */
+ H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
+
+ /* Copy the search path into the table at the specified index */
+ H5PL_paths_g[index] = path_copy;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_at() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__create_path_table
+ *
+ * Purpose: Create the collection of paths that will be searched
+ * when loading plugins.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__create_path_table(void)
+{
+ char *env_var= NULL; /* Path string from environment variable */
+ char *paths = NULL; /* Delimited paths string. Either from the
+ * environment variable or the default.
+ */
+ char *next_path = NULL; /* A path tokenized from the paths string */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Allocate memory for the path table */
+ H5PL_num_paths_g = 0;
+ H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
+ if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table")
+
+ /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it
+ * or from the default paths if it isn't set.
+ */
+ env_var = HDgetenv("HDF5_PLUGIN_PATH");
+ if (NULL == env_var)
+ paths = H5MM_strdup(H5PL_DEFAULT_PATH);
+ else
+ paths = H5MM_strdup(env_var);
+
+ if (NULL == paths)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy")
+
+ /* Separate the paths and store them */
+ /* XXX: strtok() is not thread-safe */
+ next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR);
+ while (next_path) {
+
+ /* Insert the path into the table */
+ if (H5PL__append_path(next_path) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path)
+
+ /* Get the next path from the environment string */
+ next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR);
+ } /* end while */
+
+done:
+ if (paths)
+ paths = (char *)H5MM_xfree(paths);
+
+ /* Try to clean up on errors */
+ if (FAIL == ret_value) {
+ if (H5PL_paths_g)
+ H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
+ H5PL_path_capacity_g = 0;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__create_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close_path_table
+ *
+ * Purpose: Close the collection of paths that will be searched
+ * when loading plugins.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close_path_table(void)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Free paths */
+ for (u = 0; u < H5PL_num_paths_g; u++)
+ if (H5PL_paths_g[u])
+ H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]);
+
+ /* Free path table */
+ H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
+
+ /* Reset values */
+ H5PL_num_paths_g = 0;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5PL__close_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_num_paths
+ *
+ * Purpose: Gets the number of plugin paths that have been stored.
+ *
+ * Return: Success: The number of paths
+ * Failture: Can't fail
+ *-------------------------------------------------------------------------
+ */
+unsigned
+H5PL__get_num_paths(void)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ FUNC_LEAVE_NOAPI(H5PL_num_paths_g)
+
+} /* end H5PL__get_num_paths() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__expand_path_table
+ *
+ * Purpose: Expand the path table when it's full.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__expand_path_table(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Update the capacity */
+ H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD;
+
+ /* Resize the array */
+ if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed")
+
+ /* Initialize the new memory */
+ HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *));
+
+done:
+ /* Set the path capacity back if there were problems */
+ if (FAIL == ret_value)
+ H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__expand_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__append_path
+ *
+ * Purpose: Insert a path at the end of the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__append_path(const char *path)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Insert the path at the end of the table */
+ if (H5PL__insert_at(path, H5PL_num_paths_g) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__append_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__prepend_path
+ *
+ * Purpose: Insert a path at the beginning of the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__prepend_path(const char *path)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+
+ /* Insert the path at the beginning of the table */
+ if (H5PL__insert_at(path, 0) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__prepend_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__replace_path
+ *
+ * Purpose: Replace a path at particular index in the table.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__replace_path(const char *path, unsigned int index)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Insert the path at the requested index */
+ if (H5PL__replace_at(path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__insert_path
+ *
+ * Purpose: Insert a path at particular index in the table, moving
+ * any existing paths back to make space.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__insert_path(const char *path, unsigned int index)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(path);
+ HDassert(HDstrlen(path));
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Insert the path at the requested index */
+ if (H5PL__insert_at(path, index) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__insert_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__remove_path
+ *
+ * Purpose: Remove a path at particular index in the table, freeing
+ * the path string and moving the paths down to close the gap.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__remove_path(unsigned int index)
+{
+ unsigned u; /* iterator */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(index < H5PL_path_capacity_g);
+
+ /* Check if the path at that index is set */
+ if (!H5PL_paths_g[index])
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index)
+
+ /* Delete the path */
+ H5PL_num_paths_g--;
+ H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
+
+ /* Shift the paths down to close the gap */
+ for (u = index; u < H5PL_num_paths_g; u++)
+ H5PL_paths_g[u] = H5PL_paths_g[u+1];
+
+ /* Set the (former) last path to NULL */
+ H5PL_paths_g[H5PL_num_paths_g] = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__remove_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__get_path
+ *
+ * Purpose: Get a pointer to a path at particular index in the table.
+ *
+ * Return: Success: A pointer to a path string stored in the table
+ * Failure: NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+const char *
+H5PL__get_path(unsigned int index)
+{
+ char *ret_value = NULL; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Get the path at the requested index */
+ if (index >= H5PL_num_paths_g)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index)
+
+ return H5PL_paths_g[index];
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__replace_path() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_path_table
+ *
+ * Purpose: Attempts to find a matching plugin in the file system
+ * using the paths stored in the path table.
+ *.
+ * The 'found' parameter will be set appropriately.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(plugin_info);
+
+ /* Initialize output parameters */
+ *found = FALSE;
+ *plugin_info = NULL;
+
+ /* Loop over the paths in the table, checking for an appropriate plugin */
+ for (u = 0; u < H5PL_num_paths_g; u++) {
+
+ /* Search for the plugin in this path */
+ if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u])
+
+ /* Break out if found */
+ if (*found) {
+ if (!plugin_info)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL")
+ break;
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path_table() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_path
+ *
+ * Purpose: Given a path, this function opens the directory and envokes
+ * another function to go through all files to find the right
+ * plugin library. Two function definitions are for Unix and
+ * Windows.
+ *
+ * The found parameter will be set to TRUE and the info
+ * parameter will be filled in on success.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef H5_HAVE_WIN32_API
+static herr_t
+H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
+{
+ char *path = NULL;
+ DIR *dirp = NULL; /* Directory stream */
+ struct dirent *dp = NULL; /* Directory entry */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(dir);
+ HDassert(plugin_info);
+
+ /* Initialize the found parameter */
+ *found = FALSE;
+
+ /* Open the directory */
+ if (!(dirp = HDopendir(dir)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir)
+
+ /* Iterate through all entries in the directory */
+ while (NULL != (dp = HDreaddir(dirp))) {
+
+ /* The library we are looking for should be called libxxx.so... on Unix
+ * or libxxx.xxx.dylib on Mac.
+ */
+#ifndef __CYGWIN__
+ if (!HDstrncmp(dp->d_name, "lib", (size_t)3) &&
+ (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) {
+#else
+ if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) &&
+ HDstrstr(dp->d_name, ".dll") ) {
+#endif
+
+ h5_stat_t my_stat;
+ size_t len;
+
+ /* Allocate & initialize the path name */
+ len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/;
+
+ if (NULL == (path = (char *)H5MM_calloc(len)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+
+ HDsnprintf(path, len, "%s/%s", dir, dp->d_name);
+
+ /* Get info for directory entry */
+ if (HDstat(path, &my_stat) == -1)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno))
+
+ /* If it is a directory, skip it */
+ if (S_ISDIR(my_stat.st_mode))
+ continue;
+
+ /* attempt to open the dynamic library as a filter library */
+ if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
+ if (*found)
+ HGOTO_DONE(SUCCEED)
+
+ path = (char *)H5MM_xfree(path);
+ } /* end if */
+ } /* end while */
+
+done:
+ if (dirp)
+ if (HDclosedir(dirp) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno))
+
+ path = (char *)H5MM_xfree(path);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path() */
+#else /* H5_HAVE_WIN32_API */
+static herr_t
+H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
+{
+ WIN32_FIND_DATAA fdFile;
+ HANDLE hFind = INVALID_HANDLE_VALUE;
+ char *path = NULL;
+ char service[2048];
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(dir);
+ HDassert(plugin_info);
+
+ /* Initialize the found parameter */
+ *found = FALSE;
+
+ /* Specify a file mask. *.* = We want everything! */
+ HDsprintf(service, "%s\\*.dll", dir);
+ if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory")
+
+ /* Loop over all the files */
+ do {
+ /* Ignore '.' and '..' */
+ if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) {
+
+ /* XXX: Probably just continue here and move the code below over one tab */
+
+ size_t len;
+
+ /* Allocate & initialize the path name */
+ len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1;
+
+ if (NULL == (path = (char *)H5MM_calloc(len)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
+
+ HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName);
+
+ /* Ignore directories */
+ if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ continue;
+
+ /* attempt to open the dynamic library as a filter library */
+ if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
+ if (*found)
+ HGOTO_DONE(SUCCEED)
+
+ path = (char *)H5MM_xfree(path);
+ }
+ } while (FindNextFileA(hFind, &fdFile));
+
+done:
+ if (hFind != INVALID_HANDLE_VALUE)
+ FindClose(hFind);
+ if (path)
+ path = (char *)H5MM_xfree(path);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_path() */
+#endif /* H5_HAVE_WIN32_API */
+
diff --git a/src/H5PLpkg.h b/src/H5PLpkg.h
index e356893..0d1c271 100644
--- a/src/H5PLpkg.h
+++ b/src/H5PLpkg.h
@@ -11,6 +11,12 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*
+ * Purpose: This file contains declarations which are visible only within
+ * the H5PL package. Source files outside the H5PL package should
+ * include H5PLprivate.h instead.
+ */
+
#if !(defined H5PL_FRIEND || defined H5PL_MODULE)
#error "Do not include this file outside the H5PL package!"
#endif
@@ -27,13 +33,92 @@
/* Package Private Macros */
/**************************/
-#define H5PL_MAX_PATH_NUM 16
+/* Whether to pre-load pathnames for plugin libraries */
+#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR
+
+
+/****************************/
+/* Macros for supporting */
+/* both Windows and POSIX */
+/****************************/
+
+/*******************/
+/* Windows support */
+/*******************/
+/*
+ * SPECIAL WINDOWS NOTE
+ *
+ * Some of the Win32 API functions expand to fooA or fooW depending on
+ * whether UNICODE or _UNICODE are defined. You MUST explicitly use
+ * the A version of the functions to force char * behavior until we
+ * work out a scheme for proper Windows Unicode support.
+ *
+ * If you do not do this, people will be unable to incorporate our
+ * source code into their own CMake builds if they define UNICODE.
+ */
+#ifdef H5_HAVE_WIN32_API
+
+ /* The path separator on this platform */
+# define H5PL_PATH_SEPARATOR ";"
+
+ /* Handle for dynamic library */
+# define H5PL_HANDLE HINSTANCE
+
+ /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
+# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)
+
+ /* Get the address of a symbol in dynamic library */
+# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N)
+
+ /* Close dynamic library */
+# define H5PL_CLOSE_LIB(H) FreeLibrary(H)
+
+ /* Clear error - nothing to do */
+# define H5PL_CLR_ERROR
+
+ /* maximum size for expanding env vars */
+# define H5PL_EXPAND_BUFFER_SIZE 32767
+
+ typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void);
+
+#else /* H5_HAVE_WIN32_API */
+
+ /*****************/
+ /* POSIX support */
+ /*****************/
+
+ /* The path separator on this platform */
+# define H5PL_PATH_SEPARATOR ":"
+
+ /* Handle for dynamic library */
+# define H5PL_HANDLE void *
+
+ /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
+# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY)
+
+ /* Get the address of a symbol in dynamic library */
+# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N)
+
+ /* Close dynamic library */
+# define H5PL_CLOSE_LIB(H) dlclose(H)
+
+ /* Clear error */
+# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror())
+
+ typedef const void *(*H5PL_get_plugin_info_t)(void);
+#endif /* H5_HAVE_WIN32_API */
/****************************/
/* Package Private Typedefs */
/****************************/
+/* Data used to search for plugins */
+typedef struct H5PL_search_params_t {
+ H5PL_type_t type;
+ int id;
+} H5PL_search_params_t;
+
/*****************************/
/* Package Private Variables */
@@ -44,5 +129,31 @@
/* Package Private Prototypes */
/******************************/
+/* Accessors to global variables and flags */
+H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/);
+H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask);
+
+/* Plugin search and manipulation */
+H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/);
+H5_DLL herr_t H5PL__close(H5PL_HANDLE handle);
+
+/* Plugin cache calls */
+H5_DLL herr_t H5PL__create_plugin_cache(void);
+H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/);
+H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle);
+H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
+
+/* Plugin search path calls */
+H5_DLL herr_t H5PL__create_path_table(void);
+H5_DLL herr_t H5PL__close_path_table(void);
+H5_DLL unsigned H5PL__get_num_paths(void);
+H5_DLL herr_t H5PL__append_path(const char *path);
+H5_DLL herr_t H5PL__prepend_path(const char *path);
+H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index);
+H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index);
+H5_DLL herr_t H5PL__remove_path(unsigned int index);
+H5_DLL const char *H5PL__get_path(unsigned int index);
+H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
+
#endif /* _H5PLpkg_H */
diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c
new file mode 100644
index 0000000..d826ba0
--- /dev/null
+++ b/src/H5PLplugin_cache.c
@@ -0,0 +1,307 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: Code to implement a plugin cache which stores information
+ * about plugins which have already been loaded.
+ *
+ * The plugin cache is implemented as a dynamic, global array which
+ * will grow as new plugins are added. The capacity of the cache
+ * never shrinks since plugins stay in memory once loaded.
+ *
+ * Note that this functionality has absolutely nothing to do with
+ * the metadata or chunk caches.
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5PLpkg.h" /* Plugin */
+#include "H5Zprivate.h" /* Filter pipeline */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+/* Initial capacity of the plugin cache */
+#define H5PL_INITIAL_CACHE_CAPACITY 16
+
+/* The amount to add to the capacity when the cache is full */
+#define H5PL_CACHE_CAPACITY_ADD 16
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/* Type for the list of info for opened plugin libraries */
+typedef struct H5PL_plugin_t {
+ H5PL_type_t type; /* Plugin type */
+ int id; /* ID for the plugin */
+ H5PL_HANDLE handle; /* Plugin handle */
+} H5PL_plugin_t;
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static herr_t H5PL__expand_cache(void);
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Cache for storing opened plugin libraries */
+static H5PL_plugin_t *H5PL_cache_g = NULL;
+
+/* The number of stored plugins */
+static unsigned int H5PL_num_plugins_g = 0;
+
+/* The capacity of the plugin cache */
+static unsigned int H5PL_cache_capacity_g = 0;
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__create_plugin_cache
+ *
+ * Purpose: Create the cache that will store plugins that have already
+ * been loaded.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__create_plugin_cache(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Allocate memory for the plugin cache */
+ H5PL_num_plugins_g = 0;
+
+ H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY;
+
+ if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache")
+
+done:
+ /* Try to clean up on errors */
+ if (FAIL == ret_value) {
+ if (H5PL_cache_g)
+ H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
+ H5PL_cache_capacity_g = 0;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__create_plugin_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__close_plugin_cache
+ *
+ * Purpose: Close the cache of plugins that have already been loaded,
+ * closing all the plugins contained inside.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__close_plugin_cache(hbool_t *already_closed /*out*/)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Close opened dynamic libraries */
+ if (H5PL_cache_g) {
+
+ /* Close any cached plugins */
+ for (u = 0; u < H5PL_num_plugins_g; u++)
+ H5PL__close((H5PL_cache_g[u]).handle);
+
+ /* Free the cache array */
+ H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
+ H5PL_num_plugins_g = 0;
+ H5PL_cache_capacity_g = 0;
+
+ /* Note that actually closed the table (needed by package close call) */
+ *already_closed = FALSE;
+ }
+ else
+ *already_closed = TRUE;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__close_plugin_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__expand_cache
+ *
+ * Purpose: Expand the plugin cache when it's full.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PL__expand_cache(void)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Update the capacity */
+ H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD;
+
+ /* Resize the array */
+ if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed")
+
+ /* Initialize the new memory */
+ HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t));
+
+done:
+ /* Set the cache capacity back if there were problems */
+ if (FAIL == ret_value)
+ H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__expand_cache() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__add_plugin
+ *
+ * Purpose: Add a plugin to the plugin cached.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Expand the cache if it is too small */
+ if (H5PL_num_plugins_g >= H5PL_cache_capacity_g)
+ if (H5PL__expand_cache() < 0)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache")
+
+ /* Store the plugin info and bump the # of plugins */
+ H5PL_cache_g[H5PL_num_plugins_g].type = type;
+ H5PL_cache_g[H5PL_num_plugins_g].id = id;
+ H5PL_cache_g[H5PL_num_plugins_g].handle = handle;
+
+ H5PL_num_plugins_g++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__add_plugin() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PL__find_plugin_in_cache
+ *
+ * Purpose: Attempts to find a matching plugin from the cache.
+ *
+ * The 'found' parameter will be set appropriately.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+/* See the other use of H5PL_GET_LIB_FUNC() for an explanation
+ * for why we disable -Wpedantic here.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+herr_t
+H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
+{
+ unsigned int u; /* iterator */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args - Just assert on package functions */
+ HDassert(search_params);
+ HDassert(found);
+ HDassert(plugin_info);
+
+ /* Initialize output parameters */
+ *found = FALSE;
+ *plugin_info = NULL;
+
+ /* Loop over all the plugins, looking for one that matches */
+ for (u = 0; u < H5PL_num_plugins_g; u++) {
+
+ /* If the plugin type (filter, etc.) and ID match, query the plugin for its info */
+ if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) {
+
+ H5PL_get_plugin_info_t get_plugin_info_function;
+ const H5Z_class2_t *filter_info;
+
+ /* Get the "get plugin info" function from the plugin. */
+ if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info")))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info")
+
+ /* Call the "get plugin info" function */
+ if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)()))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info")
+
+ /* Set output parameters */
+ *found = TRUE;
+ *plugin_info = filter_info;
+
+ /* No need to continue processing */
+ break;
+
+ } /* end if */
+
+ } /* end for */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PL__find_plugin_in_cache() */
+#pragma GCC diagnostic pop
+
diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h
index 9ce1fca..3b36ccd 100644
--- a/src/H5PLpublic.h
+++ b/src/H5PLpublic.h
@@ -10,8 +10,8 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* Programmer: Raymond Lu <songyulu@hdfgroup.org>
- * 13 February 2013
+/*
+ * This file contains public declarations for the H5PL module.
*/
#ifndef _H5PLpublic_H
@@ -24,31 +24,35 @@
/* Public Typedefs */
/*******************/
+/* Special string to indicate no plugin loading.
+ */
+#define H5PL_NO_PLUGIN "::"
+
/* Plugin type used by the plugin library */
typedef enum H5PL_type_t {
- H5PL_TYPE_ERROR = -1, /*error */
- H5PL_TYPE_FILTER = 0, /*filter */
- H5PL_TYPE_NONE = 1 /*this must be last! */
+ H5PL_TYPE_ERROR = -1, /* Error */
+ H5PL_TYPE_FILTER = 0, /* Filter */
+ H5PL_TYPE_NONE = 1 /* This must be last! */
} H5PL_type_t;
/* Common dynamic plugin type flags used by the set/get_loading_state functions */
-#define H5PL_FILTER_PLUGIN 0x0001
-#define H5PL_ALL_PLUGIN 0xFFFF
+#define H5PL_FILTER_PLUGIN 0x0001
+#define H5PL_ALL_PLUGIN 0xFFFF
#ifdef __cplusplus
extern "C" {
#endif
/* plugin state */
-H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type);
-H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/);
-H5_DLL herr_t H5PLappend(const char *plugin_path);
-H5_DLL herr_t H5PLprepend(const char *plugin_path);
-H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index);
-H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index);
+H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask);
+H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/);
+H5_DLL herr_t H5PLappend(const char *search_path);
+H5_DLL herr_t H5PLprepend(const char *search_path);
+H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index);
+H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index);
H5_DLL herr_t H5PLremove(unsigned int index);
-H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size);
-H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/);
+H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size);
+H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/);
#ifdef __cplusplus
}
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 55b3877..854b1ef 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -166,7 +166,7 @@ typedef enum H5D_mpio_no_collective_cause_t {
H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08,
H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10,
H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x20,
- H5D_MPIO_FILTERS = 0x40
+ H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x40
} H5D_mpio_no_collective_cause_t;
/********************/
diff --git a/src/H5R.c b/src/H5R.c
index 73c1d55..b000183 100644
--- a/src/H5R.c
+++ b/src/H5R.c
@@ -105,7 +105,7 @@ H5R__init_package(void)
/* Initialize the atom group for the file IDs */
if(H5I_register_type(H5I_REFERENCE_CLS) < 0)
- HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface")
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface")
/* Mark "top" of interface as initialized, too */
H5R_top_package_initialize_s = TRUE;
@@ -141,10 +141,10 @@ H5R_top_term_package(void)
FUNC_ENTER_NOAPI_NOINIT_NOERR
if(H5R_top_package_initialize_s) {
- if(H5I_nmembers(H5I_REFERENCE) > 0) {
- (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE);
+ if(H5I_nmembers(H5I_REFERENCE) > 0) {
+ (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE);
n++; /*H5I*/
- } /* end if */
+ } /* end if */
/* Mark closed */
if(0 == n)
@@ -741,7 +741,7 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTCREATE, FAIL, "unable to create dataspace")
/* Atomize */
- if((ret_value = H5I_register (H5I_DATASPACE, space, TRUE)) < 0)
+ if((ret_value = H5I_register(H5I_DATASPACE, space, TRUE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace atom")
done:
diff --git a/src/H5err.txt b/src/H5err.txt
index 3f5801f..d771956 100644
--- a/src/H5err.txt
+++ b/src/H5err.txt
@@ -243,6 +243,8 @@ MINOR, LINK, H5E_CANTSORT, Can't sort objects
MINOR, MPI, H5E_MPI, Some MPI function failed
MINOR, MPI, H5E_MPIERRSTR, MPI Error String
MINOR, MPI, H5E_CANTRECV, Can't receive data
+MINOR, MPI, H5E_CANTGATHER, Can't gather data
+MINOR, MPI, H5E_NO_INDEPENDENT, Can't perform independent IO
# Heap errors
MINOR, HEAP, H5E_CANTRESTORE, Can't restore condition
diff --git a/src/H5private.h b/src/H5private.h
index c588154..ca4ebcf 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -495,6 +495,15 @@
# define H5_POSIX_MAX_IO_BYTES SSIZET_MAX
#endif
+/* POSIX I/O mode used as the third parameter to open/_open
+ * when creating a new file (O_CREAT is set).
+ */
+#if defined(H5_HAVE_WIN32_API)
+# define H5_POSIX_CREATE_MODE_RW (_S_IREAD | _S_IWRITE)
+#else
+# define H5_POSIX_CREATE_MODE_RW 0666
+#endif
+
/*
* A macro to portably increment enumerated types.
*/
@@ -1117,11 +1126,7 @@ typedef off_t h5_stat_size_t;
#define HDnanosleep(N, O) nanosleep(N, O)
#endif /* HDnanosleep */
#ifndef HDopen
- #ifdef _O_BINARY
- #define HDopen(S,F,M) open(S,F|_O_BINARY,M)
- #else
- #define HDopen(S,F,M) open(S,F,M)
- #endif
+ #define HDopen(F,...) open(F,__VA_ARGS__)
#endif /* HDopen */
#ifndef HDopendir
#define HDopendir(S) opendir(S)
diff --git a/src/H5system.c b/src/H5system.c
index 7e25540..a1cdf19 100644
--- a/src/H5system.c
+++ b/src/H5system.c
@@ -1237,3 +1237,49 @@ H5_get_time(void)
} /* end H5_get_time() */
+#ifdef H5_HAVE_WIN32_API
+
+#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5_expand_windows_env_vars()
+ *
+ * Purpose: Replaces windows environment variables of the form %foo%
+ * with user-specific values.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5_expand_windows_env_vars(char **env_var)
+{
+ long n_chars = 0;
+ char *temp_buf = NULL;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Allocate buffer for expanded environment variable string */
+ if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE)))
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path")
+
+ /* Expand the environment variable string */
+ if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long")
+
+ if (0 == n_chars)
+ HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path")
+
+ *env_var = (char *)H5MM_xfree(*env_var);
+ *env_var = temp_buf;
+
+done:
+ if (FAIL == ret_value && temp_buf)
+ temp_buf = (char *)H5MM_xfree(temp_buf);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5_expand_windows_env_vars() */
+#endif /* H5_HAVE_WIN32_API */
+
diff --git a/src/H5trace.c b/src/H5trace.c
index 9fb8a72..930002f 100644
--- a/src/H5trace.c
+++ b/src/H5trace.c
@@ -621,10 +621,6 @@ H5_trace(const double *returning, const char *func, const char *type, ...)
fprintf(out, "%sH5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET", flag_already_displayed ? " | " : "");
flag_already_displayed = TRUE;
} /* end if */
- if(nocol_cause_mode & H5D_MPIO_FILTERS) {
- fprintf(out, "%sH5D_MPIO_FILTERS", flag_already_displayed ? " | " : "");
- flag_already_displayed = TRUE;
- } /* end if */
/* Display '<none>' if there's no flags set */
if(!flag_already_displayed)
diff --git a/src/H5win32defs.h b/src/H5win32defs.h
index 0149faa..4522228 100644
--- a/src/H5win32defs.h
+++ b/src/H5win32defs.h
@@ -48,9 +48,13 @@ typedef __int64 h5_stat_size_t;
#define HDnanosleep(N, O) Wnanosleep(N, O)
#define HDoff_t __int64
/* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL
- * transformations when performing I/O.
+ * transformations when performing I/O. Note that this will
+ * produce Unix-style text files, though.
+ *
+ * Also note that the variadic macro is using a VC++ extension
+ * where the comma is dropped if nothing is passed to the ellipsis.
*/
-#define HDopen(S,F,M) _open(S,F|_O_BINARY,M)
+#define HDopen(S,F,...) _open(S, F | _O_BINARY, __VA_ARGS__)
#define HDread(F,M,Z) _read(F,M,Z)
#define HDrmdir(S) _rmdir(S)
#define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,(Z>1?Z:2))
@@ -115,6 +119,7 @@ extern "C" {
H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...);
H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap);
H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem);
+ H5_DLL herr_t H5_expand_windows_env_vars(char **env_var);
/* Round functions only needed for VS2012 and earlier.
* They are always built to ensure they don't go stale and
diff --git a/src/Makefile.am b/src/Makefile.am
index 0b664a7..9a64717 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Pgcpl.c H5Pint.c \
H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \
H5PB.c \
- H5PL.c \
+ H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \
H5R.c H5Rdeprec.c \
H5UC.c \
H5RS.c \