Lots of bug fixes, almost working with examples/vpic-query.c

8 files changed, 630 insertions, 325 deletions

diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c
index a933e92..480b3e4 100644
--- a/src/H5Pdcpl.c
+++ b/src/H5Pdcpl.c
@@ -3657,9 +3657,8 @@ H5Pset_index_plugin(hid_t plist_id, unsigned plugin_id)
     if (plugin_id > H5X_PLUGIN_MAX)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid plugin identification number");
 
-    /* Get the property list structure */
-    if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
-        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+            HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
 
     plugin = plugin_id;
 
diff --git a/src/H5Q.c b/src/H5Q.c
index ca3c50e..0f5b1d3 100644
--- a/src/H5Q.c
+++ b/src/H5Q.c
@@ -283,6 +283,10 @@ static herr_t H5Q__view_combine(H5Q_combine_op_t combine_op, H5Q_view_t *view1,
 static herr_t H5Q__view_write(H5G_t *grp, H5Q_view_t *view);
 static herr_t H5Q__view_free(H5Q_view_t *view);
 
+/* Parallel query support functions */
+static hbool_t H5Q__contains_data_match(H5Q_t *query);
+static hbool_t H5Q__contains_link_match(H5Q_t *query, char *obj_name, hbool_t previous);
+
 
 static H5_INLINE void
 H5Q__encode_memcpy(unsigned char **buf_ptr, size_t *nalloc, const void *data,
@@ -1383,7 +1387,9 @@ H5Qapply_atom(hid_t query_id, hbool_t *result, ...)
 {
     H5Q_t *query = NULL;
     H5T_t *native_type = NULL;
+    hbool_t summation_result = false;
     herr_t ret_value;
+    unsigned local_result, global_result;
     va_list ap;
 
     FUNC_ENTER_API(FAIL)
@@ -1458,6 +1464,14 @@ H5Qapply_atom(hid_t query_id, hbool_t *result, ...)
     }
 
 done:
+
+#ifdef H5_HAVE_PARALLEL
+    local_result = (unsigned)(*result);
+    summation_result = H5Q__contains_data_match(query);
+    global_result = H5Xallreduce_unsigned_status(local_result, summation_result);
+    *result = (hbool_t)(global_result);
+#endif
+
     if (native_type)
         H5T_close(native_type);
 
@@ -1516,7 +1530,6 @@ H5Q__apply_atom(const H5Q_t *query, hbool_t *result, va_list ap)
 
     HDassert(query);
     HDassert(result);
-    HDassert(query->is_combined == FALSE);
 
     va_copy(aq, ap);
 
@@ -1864,8 +1877,11 @@ hid_t
 H5Qapply(hid_t loc_id, hid_t query_id, unsigned *result, hid_t vcpl_id)
 {
     H5Q_t *query = NULL;
+    hbool_t summation_result = false;
     H5G_t *ret_grp;
     hid_t ret_value;
+    unsigned local_result;
+    unsigned global_result;
 
     FUNC_ENTER_API(FAIL)
     H5TRACE4("i", "ii*Iui", loc_id, query_id, result, vcpl_id);
@@ -1892,6 +1908,14 @@ H5Qapply(hid_t loc_id, hid_t query_id, unsigned *result, hid_t vcpl_id)
         HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register group");
 
 done:
+
+#ifdef H5_HAVE_PARALLEL
+    local_result = *result;
+    summation_result = H5Q__contains_data_match(query);
+    global_result = H5Xallreduce_unsigned_status(local_result, summation_result);
+    *result = global_result;
+#endif
+
     FUNC_LEAVE_API(ret_value)
 } /* end H5Qapply() */
 
@@ -2186,6 +2210,13 @@ done:
 } /* H5Q__apply_index() */
 
 
+typedef struct lnk_entry_list {
+    char *link_name;
+    H5Q_combine_op_t op;
+    struct lnk_entry_list *next;
+} lnk_entry_list_t;
+
+
 static char *
 whitespace(int level)
 {
@@ -2361,7 +2392,6 @@ show_query(H5Q_t *query, int indent)
     }
 }
 
-
 static int visualize_query = 0;
 
 void
@@ -2370,6 +2400,77 @@ H5Q_enable_visualize_query(void)
     visualize_query++;
 }
 
+static hbool_t
+H5Q__contains_data_match(H5Q_t *query)
+{
+    H5Q_type_t query_type;
+    H5Q_combine_op_t op_type;
+    H5T_t *d_type;
+    H5T_class_t data_class;
+    hbool_t have_match = false;
+
+    if (FAIL == H5Q_get_type(query, &query_type)) {
+        printf("H5Q_get_type returned error!\n");
+    }
+    else {
+        switch(query_type) {
+	case H5Q_TYPE_DATA_ELEM:
+	  return true;
+          break;
+        case H5Q_TYPE_ATTR_VALUE:
+	case H5Q_TYPE_ATTR_NAME:
+	case H5Q_TYPE_LINK_NAME:
+          break;
+        case H5Q_TYPE_MISC:
+	  if (query->is_combined) {
+	    have_match = H5Q__contains_data_match(query->query.combine.l_query);
+	    have_match |= H5Q__contains_data_match(query->query.combine.r_query);
+	  }
+	}
+    }
+    return have_match;
+}
+
+static hbool_t
+H5Q__contains_link_match(H5Q_t *query, char *obj_name, hbool_t previous)
+{
+    if (query->is_combined == false) {
+      if ((query->query.select.type == H5Q_TYPE_LINK_NAME) &&
+	  (query->query.select.match_op == H5Q_MATCH_EQUAL)) {
+	char *link_name = query->query.select.elem.link_name.name;
+	char *slash;
+	char *dataset;
+	slash = strrchr(link_name, '/');
+	/* If groups are utilized, name will be the fully qualified name which
+	 * includes slashes ('/').  If user isn't specific to a group with the
+	 * query, then we'll attempt to match the non-group-specific names
+	 */
+	if ((dataset = strrchr(obj_name, '/')) != NULL) {
+	  if (slash == NULL) {	/* General User Query */
+	    dataset++;
+	    if (strcmp(link_name, dataset) == 0)
+	      return TRUE;
+	    return FALSE;
+	  }
+          else { /* Specific User Query */
+	    if (strcmp(link_name, obj_name) == 0)
+	      return TRUE;
+	    return FALSE;
+	  }
+	}
+	else {
+	  if (strcmp(link_name, obj_name) == 0) 
+	    return TRUE;
+	  return FALSE;
+	}
+      }
+    }
+
+    return previous;
+}
+
+
+
 /*-------------------------------------------------------------------------
  * Function:    H5Q__apply_iterate
  *
@@ -2382,12 +2483,13 @@ H5Q_enable_visualize_query(void)
 static herr_t
 H5Q__apply_iterate(hid_t oid, const char *name, const H5O_info_t *oinfo, void *op_data)
 {
-    static int count = 0;
+    static hbool_t restrict_link_targets = FALSE;
+    static int misc_count = 0;
+
     H5Q_apply_arg_t *args = (H5Q_apply_arg_t *) op_data;
     H5Q_type_t query_type;
     herr_t ret_value = SUCCEED; /* Return value */
 
-
     FUNC_ENTER_NOAPI_NOINIT
 
     HDassert(args->query);
@@ -2405,12 +2507,14 @@ H5Q__apply_iterate(hid_t oid, const char *name, const H5O_info_t *oinfo, void *o
         H5Q_apply_arg_t args1, args2;
         H5Q_view_t view1 = H5Q_VIEW_INITIALIZER(view1), view2 = H5Q_VIEW_INITIALIZER(view2);
         unsigned result1 = 0, result2 = 0;
+
 	if (visualize_query > 0) {
-            show_query(args->query->query.combine.l_query, 0);
-            show_query(args->query->query.combine.r_query, 0);
+            puts("");
+            show_query(args->query, 0);
 	    puts("--------------\n");
 	    visualize_query = 0;
 	}
+
         if (FAIL == H5Q_get_combine_op(args->query, &op_type))
             HGOTO_ERROR(H5E_QUERY, H5E_CANTGET, FAIL, "unable to get combine op");
 
@@ -2422,11 +2526,22 @@ H5Q__apply_iterate(hid_t oid, const char *name, const H5O_info_t *oinfo, void *o
         args2.result = &result2;
         args2.view = &view2;
 
-        if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args1))
-            HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
-        if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args2))
-            HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
-
+        if (oinfo->type != H5O_TYPE_DATASET) {
+            if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args1))
+                HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
+            if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args2))
+                HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
+        }
+	else {
+            hbool_t match1 = H5Q__contains_link_match(args1.query, name, true);
+            hbool_t match2 = H5Q__contains_link_match(args2.query, name, match1);
+            if (match1 || match2) {
+                if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args1))
+                    HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
+                if (FAIL == H5Q__apply_iterate(oid, name, oinfo, &args2))
+                    HGOTO_ERROR(H5E_QUERY, H5E_CANTCOMPARE, FAIL, "unable to apply query");
+            }
+ 	}
         if (FAIL == H5Q__view_combine(op_type, &view1, &view2, result1, result2,
                 args->view, args->result))
             HGOTO_ERROR(H5E_QUERY, H5E_CANTMERGE, FAIL, "unable to merge results");
@@ -3029,8 +3144,12 @@ H5Q__view_write(H5G_t *grp, H5Q_view_t *view)
 {
     H5G_loc_t loc;
     H5D_t *dset = NULL;
+    hid_t dataset = -1;
+    hid_t viewspace = -1;
+    hid_t gid = -1;
     H5S_t *mem_space = NULL;
     H5S_t *space = NULL;
+    hsize_t max_unlim[1] = {H5S_UNLIMITED};	/* Maximum size of dataspace (unlimited) */
     H5Q_ref_head_t *refs[H5Q_VIEW_REF_NTYPES] = { &view->reg_refs, &view->obj_refs,
             &view->attr_refs };
     const char *dset_name[H5Q_VIEW_REF_NTYPES] = { H5Q_VIEW_REF_REG_NAME,
@@ -3053,12 +3172,13 @@ H5Q__view_write(H5G_t *grp, H5Q_view_t *view)
         H5Q_ref_entry_t *ref_entry = NULL;
         hsize_t n_elem = refs[i]->n_elem;
         hsize_t start = 0;
+	hsize_t *maxdims = NULL;
 
         if (!n_elem)
             continue;
 
         /* Create dataspace */
-        if (NULL == (space = H5S_create_simple(1, &n_elem, NULL)))
+        if (NULL == (space = H5S_create_simple(1, &n_elem, maxdims)))
             HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "unable to create dataspace");
 
         /* Create the new dataset & get its ID */
@@ -3071,8 +3191,7 @@ H5Q__view_write(H5G_t *grp, H5Q_view_t *view)
         /* Iterate over reference entries in view */
         H5Q_QUEUE_FOREACH(ref_entry, refs[i], entry) {
             hsize_t count = 1;
-
-            if (NULL == (mem_space = H5S_create_simple(1, &count, NULL)))
+            if (NULL == (mem_space = H5S_create_simple(1, &count, maxdims)))
                 HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "unable to create dataspace");
             if (FAIL == H5S_select_hyperslab(space, H5S_SELECT_SET, &start, NULL, &count, NULL))
                 HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to set hyperslab selection")
diff --git a/src/H5R.c b/src/H5R.c
index 5358dfa..ef7c8f7 100644
--- a/src/H5R.c
+++ b/src/H5R.c
@@ -1108,6 +1108,7 @@ H5Rcopy(href_t ref)
     href_t ret_value;
 
     FUNC_ENTER_API(NULL)
+    H5TRACE1("r", "r", ref);
 
     /* Check args */
     if(!ref)
@@ -2129,6 +2130,7 @@ H5Rencode(href_t ref, void *buf, size_t *nalloc)
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_API(FAIL)
+    H5TRACE3("e", "r*x*z", ref, buf, nalloc);
 
     /* Check argument and retrieve object */
     if(nalloc == NULL)
@@ -2201,6 +2203,7 @@ H5Rdecode(const void *buf)
     href_t ret_value; /* Return value */
 
     FUNC_ENTER_API(NULL)
+    H5TRACE1("r", "*x", buf);
 
     /* Check args */
     if(buf == NULL)
diff --git a/src/H5Shyper.c b/src/H5Shyper.c
index 90047ec..774e79d 100644
--- a/src/H5Shyper.c
+++ b/src/H5Shyper.c
@@ -8034,6 +8034,15 @@ H5S_combine_select(H5S_t *space1, H5S_seloper_t op, H5S_t *space2)
         if(H5S_hyper_generate_spans(space1) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, NULL, "dataspace does not have span tree")
 
+    if (space2->select.type->type == H5S_SEL_NONE) {
+        if (op == H5S_SELECT_AND)
+            ret_value = new_space = H5S_copy(space2, FALSE, TRUE);
+	else if (op == H5S_SELECT_OR)
+            ret_value = new_space = H5S_copy(space1, FALSE, TRUE);
+	goto done;
+    }
+
+    
     if(NULL == space2->select.sel_info.hslab->span_lst) {
         hsize_t tmp_start[H5S_MAX_RANK];
         hsize_t tmp_stride[H5S_MAX_RANK];
diff --git a/src/H5X.c b/src/H5X.c
index 08cc7b7..2a6e927 100644
--- a/src/H5X.c
+++ b/src/H5X.c
@@ -1017,7 +1017,7 @@ int getLocalInfo(int hostid)
 }
 
 static
-void gather_topology_info(void)
+void gather_topology_info(MPI_Comm comm)
 {
     int isInitialized = 0;
     int hostid = gethostid();
@@ -1026,8 +1026,8 @@ void gather_topology_info(void)
        enable_parallel_queries = 0;
        return;
     }
-    MPI_Comm_rank(MPI_COMM_WORLD, &g_mpi_rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &g_mpi_size);
+    MPI_Comm_rank(comm, &g_mpi_rank);
+    MPI_Comm_size(comm, &g_mpi_size);
     if (g_mpi_size == 1) {
       g_group_peer_count = 1;
       g_local_peer_count = 1;
@@ -1041,7 +1041,7 @@ void gather_topology_info(void)
 }
 
 static 
-herr_t create_group_comms(void)
+herr_t create_group_comms(MPI_Comm comm)
 {
     herr_t ret_value = SUCCEED; /* Return value */
     int query_rank = -1;
@@ -1087,11 +1087,17 @@ herr_t create_group_comms(void)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5Xinitialize_parallel_query(void)
+H5Xinitialize_parallel_query()
 {
     static int initialized = 0;
-    gather_topology_info();
-    return create_group_comms();
+    herr_t ret = SUCCEED;
+    MPI_Comm comm = MPI_COMM_WORLD;
+    if (!initialized) {
+       gather_topology_info(comm);
+       ret = create_group_comms(comm);
+       if (ret == SUCCEED) initialized++;
+    }
+    return ret;
 }
 
 int
@@ -1112,6 +1118,31 @@ H5Xparallel_size(void)
     return g_mpi_size;
 }
 
+unsigned
+H5Xallreduce_unsigned_status(unsigned status, hbool_t do_summation)
+{
+    unsigned global_status = 0;
+    unsigned local_status = status;
+    if ((g_mpi_size > 1) && do_summation) {
+        if (MPI_SUCCESS == MPI_Allreduce(&local_status, &global_status, 1, MPI_UNSIGNED, MPI_SUM, query_group_comm))
+            return global_status;
+    }
+    return status;
+}
+
+int
+H5Xallreduce_int_status(int status, hbool_t do_summation)
+{
+    int global_status = 0;
+    int local_status = status;
+    if ((g_mpi_size > 1) && do_summation) {    
+        if (MPI_SUCCESS == MPI_Allreduce(&local_status, &global_status, 1, MPI_INT, MPI_SUM, query_group_comm))
+            return global_status;
+    }
+    return status;
+}
+
+
 herr_t
 H5Xallgather_by_size(void *alldata, int nelems, int typesize)
 {
diff --git a/src/H5Xdummy.c b/src/H5Xdummy.c
index 1a69dbf..e943c81 100644
--- a/src/H5Xdummy.c
+++ b/src/H5Xdummy.c
@@ -184,8 +184,11 @@ done:
         H5Tclose(type_id);
     if (space_id != FAIL)
         H5Sclose(space_id);
-    if (ret_value == FAIL)
+    if (ret_value == FAIL) {
+        *buf = NULL;
+	*buf_size = 0;
         H5MM_free(data);
+    }
     FUNC_LEAVE_NOAPI(ret_value)
 }
 
@@ -268,7 +271,7 @@ H5X_dummy_create(hid_t dataset_id, hid_t H5_ATTR_UNUSED xcpl_id, hid_t H5_ATTR_U
 done:
     if (FAIL != file_id)
         H5Fclose(file_id);
-    H5MM_free(buf);
+    if (buf) H5MM_free(buf);
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5X_dummy_create() */
 
@@ -383,6 +386,7 @@ H5X_dummy_close(void *idx_handle)
     if (FAIL == H5Dclose(dummy->idx_anon_id))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close anonymous dataset for index");
 
+    if (dummy->idx_token != NULL)
     H5MM_free(dummy->idx_token);
     H5MM_free(dummy);
 
@@ -570,7 +574,7 @@ H5X_dummy_query(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id, hid_t query
             (int) H5Sget_select_npoints(ret_value));
 
 done:
-    H5MM_free(buf);
+    if (buf) H5MM_free(buf);
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5X_dummy_query() */
 
diff --git a/src/H5Xfastbit.c b/src/H5Xfastbit.c
index 4dca079..5f49126 100644
--- a/src/H5Xfastbit.c
+++ b/src/H5Xfastbit.c
@@ -51,6 +51,7 @@
 /****************/
 /* Local Macros */
 /****************/
+#define SHOW_TIME
 // #define H5X_FASTBIT_DEBUG
 
 #ifdef H5X_FASTBIT_DEBUG
@@ -71,9 +72,11 @@
 #define H5X_FASTBIT_MAX_NAME 1024
 
 struct fastbit_meta_collection {
-    int64_t nkeys;
-    int64_t noffsets;
-    int64_t nbitmaps;
+ /* hsize_t base_offset;   // We'll eventually need this too... */
+    hsize_t data_length;
+    hsize_t nkeys;
+    hsize_t noffsets;
+    hsize_t nbitmaps;
 };
 
 /******************/
@@ -107,12 +110,15 @@ typedef struct H5X_fastbit_t {
     char column_name[H5X_FASTBIT_MAX_NAME];
 
     /* For parallel query/indexing support */
-    int nranks;
+    int nblocks;
     hid_t filespace_id; 
     hid_t memspace_id;
     hid_t index_info_group_id;
+    int64_t blocksize;
+    int64_t nelmts;
 
     hbool_t idx_reconstructed;
+    hbool_t idx_multiblock;
 } H5X_fastbit_t;
 
 struct H5X_fastbit_scatter_info {
@@ -324,7 +330,7 @@ H5X__get_fastbit(hid_t dataset_id)
 
 
 static void
-dump_as_bytes(char *entry, int64_t length, void *data)
+dump_as_bytes(char *entry, hsize_t length, void *data)
 {
   static int mpi_rank = -1;
   char filename[80];
@@ -332,6 +338,7 @@ dump_as_bytes(char *entry, int64_t length, void *data)
   unsigned char *show = (char *)data;
   int i, k;
   int rows = (int)(length/10);
+  int lastrow = (int)(length % 10);
   if (mpi_rank < 0) mpi_rank = H5Xparallel_rank();
   sprintf(filename, "%s-dump-%d.txt", entry, mpi_rank);
   save_dump = fopen(filename, "a+");
@@ -342,6 +349,12 @@ dump_as_bytes(char *entry, int64_t length, void *data)
 	   show[1],show[2],show[3],show[4],show[5],show[6],show[7],show[8],show[9]);
     show += 10;
   }
+  if (lastrow) {
+    fprintf(save_dump,"%4d\t%02x", rows, show[0]);
+    for(i=1; i< lastrow; i++)
+      fprintf(save_dump," %02x", show[i]);
+    fprintf(save_dump,"\n");
+  }
   fclose(save_dump);
 }
 
@@ -395,12 +408,13 @@ H5X__fastbit_init(hid_t dataset_id)
     fastbit->dataset_name = strdup(dataset_name);
     sprintf(fastbit->column_name, "array%u", H5X__fastbit_hash_string(dataset_name));
 
-    fastbit->nranks = 0;
+    fastbit->nblocks = 0;
     fastbit->filespace_id = FAIL;
     fastbit->memspace_id = FAIL;
     fastbit->index_info_group_id = FAIL;
 
     fastbit->idx_reconstructed = FALSE;
+    fastbit->idx_multiblock = FALSE;
 
     /* Initialize FastBit (no config file for now) */
     fastbit_init(NULL);
@@ -597,7 +611,8 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
     herr_t ret_value = SUCCEED; /* Return value */
     hid_t type_id = FAIL, space_id = FAIL;
     hid_t mem_space = H5S_ALL;
-    hid_t file_space = H5P_DEFAULT;;
+    hid_t file_space = H5P_DEFAULT;
+    hid_t dxpl_id  = H5P_DEFAULT;
 
     size_t nelmts, elmt_size;
     void *data = NULL;
@@ -618,6 +633,7 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
 	int i, ds_ndims = 0;
         hsize_t *start = NULL, *count = NULL, *block = NULL, *stride = NULL;
 	H5X_fastbit_t *fastbit = H5X__get_fastbit(dataset_id);
+	if (fastbit->filespace_id < 0) fastbit->filespace_id = space_id;
 	ds_ndims = H5Xslab_set(fastbit->filespace_id, &start, &count, &stride, &block);
 	file_space = fastbit->filespace_id;
 
@@ -645,6 +661,11 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
 	 * from this pointer.  As a result, we only need to free the start pointer.
 	 */
 	free(start);
+
+        if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "unable to create xfer property");
+        if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "unable to set MPIO_COLLECTIVE property");
     } 
     else {
         /* Get space info */
@@ -656,6 +677,7 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
         if (NULL == (data = H5MM_malloc(data_size)))
             HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
     }
+
 #else
     if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
         HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
@@ -665,11 +687,13 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
     if (NULL == (data = H5MM_malloc(data_size)))
         HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
 #endif      
-
     /* Read data from dataset */
-    if (FAIL == H5Dread(dataset_id, type_id, mem_space, file_space, H5P_DEFAULT, data))
+    if (FAIL == H5Dread(dataset_id, type_id, mem_space, file_space, dxpl_id, data))
         HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
 
+    if (dxpl_id != H5P_DEFAULT)
+        H5Pclose(dxpl_id);
+
     /* Convert type to FastBit type */
     if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
@@ -804,9 +828,9 @@ H5X__fastbit_build_index(H5X_fastbit_t *fastbit)
         HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
 
     if (sequence_id == -1) {	/* Disabled for now */
-      int64_t key_array_size = fastbit->nkeys * sizeof(double);
-      int64_t offset_array_size = fastbit->noffsets * sizeof(int64_t);
-      int64_t bitmap_array_size = fastbit->nbitmaps * sizeof(uint32_t);
+      key_array_size = fastbit->nkeys * sizeof(double);
+      offset_array_size = fastbit->noffsets * sizeof(int64_t);
+      bitmap_array_size = fastbit->nbitmaps * sizeof(uint32_t);
 
       sprintf(log_entry, "seq-keys-save-%s", fastbit->column_name);
       dump_as_bytes(log_entry, key_array_size, fastbit->keys);
@@ -867,6 +891,8 @@ H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbi
         count = (hsize_t)gatherInfo[mpi_rank].nkeys;
         if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
             HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
+	if (H5Sselect_all(_memspace_id) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
         if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
             HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
         if (FAIL == (fastbit->keys_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
@@ -881,6 +907,8 @@ H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbi
         count = (hsize_t)gatherInfo[mpi_rank].noffsets;
         if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
             HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
+	if (H5Sselect_all(_memspace_id) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
         if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
             HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
         if (FAIL == (fastbit->offsets_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
@@ -895,6 +923,8 @@ H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbi
         count = (hsize_t)gatherInfo[mpi_rank].nbitmaps;
         if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
             HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
+	if (H5Sselect_all(_memspace_id) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
         if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
             HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
         if (FAIL == (fastbit->bitmaps_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
@@ -907,6 +937,8 @@ H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbi
     else if (typeIndex == INFO_) {
         if (FAIL == (_memspace_id = H5Screate_simple(1, array_size, NULL)))
             HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
+	if (H5Sselect_all(_memspace_id) < 0)
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
         if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, array_size, NULL)) < 0 )
             HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
         if (FAIL == (fastbit->index_info_group_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
@@ -957,10 +989,10 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
     herr_t ret_value = SUCCEED; /* Return value */
     hid_t dxpl_id = H5P_DEFAULT;
     int i, mpi_rank, mpi_size;
-    int64_t nkeys_totalsize = 0, nkeys_offset = 0;
-    int64_t noffsets_totalsize = 0, noffsets_offset = 0;
-    int64_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
-    int64_t indexinfo_totalsize = 0, indexinfo_offset = 0;
+    hsize_t nkeys_totalsize = 0, nkeys_offset = 0;
+    hsize_t noffsets_totalsize = 0, noffsets_offset = 0;
+    hsize_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
+    hsize_t indexinfo_totalsize = 0, indexinfo_offset = 0;
     struct fastbit_meta_collection *gatherInfo = NULL;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -999,20 +1031,21 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
      *    +-------------+
      *    
      */
-    /* Allocate storage for 3 elements per rank */
-    indexinfo_totalsize = (3 * sizeof(int64_t) * mpi_size);
+    /* Allocate storage for 4 elements per rank */
+    indexinfo_totalsize = (hsize_t)(sizeof(struct fastbit_meta_collection) * (hsize_t)(mpi_size));
     if (NULL == (gatherInfo = (struct fastbit_meta_collection *)H5MM_malloc(indexinfo_totalsize)))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate nkeys array");
 
-    gatherInfo[mpi_rank].nkeys    = (int64_t)(fastbit->nkeys);
-    gatherInfo[mpi_rank].noffsets = (int64_t)(fastbit->noffsets);
-    gatherInfo[mpi_rank].nbitmaps = (int64_t)(fastbit->nbitmaps);
+    gatherInfo[mpi_rank].data_length = (hsize_t)H5Sget_select_npoints(fastbit->memspace_id);
+    gatherInfo[mpi_rank].nkeys    = (hsize_t)(fastbit->nkeys);
+    gatherInfo[mpi_rank].noffsets = (hsize_t)(fastbit->noffsets);
+    gatherInfo[mpi_rank].nbitmaps = (hsize_t)(fastbit->nbitmaps);
 
     /* Exchange info with all MPI ranks */
     /* We could use allreduce here if we only needed the total sizes
      * but we also want to figure out the offsets (for the hyperslab selection).
      */
-    if (H5Xallgather_by_size(gatherInfo, 3, sizeof(int64_t)) != SUCCEED)
+    if (H5Xallgather_by_size(gatherInfo, 4, sizeof(int64_t)) != SUCCEED)
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't gather nkeys array");
 
     /* Gather the total sizes and offset info */
@@ -1022,13 +1055,15 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
            noffsets_offset = noffsets_totalsize;
            nbitmaps_offset = nbitmaps_totalsize;
         }
-        nkeys_totalsize += gatherInfo[i].nkeys;
-        noffsets_totalsize += gatherInfo[i].noffsets;
-        nbitmaps_totalsize += gatherInfo[i].nbitmaps;
+        nkeys_totalsize += gatherInfo[i].nkeys+1;
+        noffsets_totalsize += gatherInfo[i].noffsets+1;
+        nbitmaps_totalsize += gatherInfo[i].nbitmaps+1;
     }
 
     /* Save the exchanged info for future use */
-    fastbit->nranks = mpi_size;
+    fastbit->nblocks = mpi_size;
+    fastbit->idx_multiblock = TRUE;
+    fastbit->nelmts = gatherInfo[mpi_rank].data_length;
 
     /* Create a transfer property to utilize MPI-IO */
     if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 )
@@ -1036,19 +1071,10 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
     if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 )
        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't set dataset_xfer to MPIO_COLLECTIVE");
 
-
     /* -------------- */
     /*  INFOGROUP     */
     /* -------------- */       
 
-    /* We'll write out the entire 'gatherInfo' array into a dataset for later reuse */
-    if (fastbit->index_info_group_id != FAIL) {
-        if (FAIL == H5Odecr_refcount(fastbit->index_info_group_id))
-            HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't decrement dataset refcount");
-        if (FAIL == H5Dclose(fastbit->index_info_group_id))
-            HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close dataset");
-    }
-
     /* Filespace descriptors */
     indexinfo_totalsize  /= sizeof(int64_t); /* Previously, was the malloc size in bytes */
                                              /* Fixed to be the total number of elements */
@@ -1062,18 +1088,24 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
     offsets_memspace_id  = H5Screate_simple(1, &gatherInfo[mpi_rank].noffsets, NULL);
     bitmaps_memspace_id  = H5Screate_simple(1, &gatherInfo[mpi_rank].nbitmaps, NULL);
 
+    if (H5Sselect_all(keys_memspace_id) < 0)
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
+    if (H5Sselect_all(offsets_memspace_id) < 0)
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
+    if (H5Sselect_all(bitmaps_memspace_id) < 0)
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
+
     /* Hyperslab selections */
-    if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, &nkeys_offset, NULL,
-			     &gatherInfo[mpi_rank].nkeys, NULL)) < 0 )
+    if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, (hsize_t *)(&nkeys_offset), NULL,
+			     (hsize_t *)(&gatherInfo[mpi_rank].nkeys), NULL)) < 0 )
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-    if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, &noffsets_offset, NULL,
-			     &gatherInfo[mpi_rank].noffsets, NULL)) < 0 )
+    if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, (hsize_t *)(&noffsets_offset), NULL,
+			     (hsize_t *)(&gatherInfo[mpi_rank].noffsets), NULL)) < 0 )
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
     if ((H5Sselect_hyperslab(bitmaps_filespace_id, H5S_SELECT_SET, &nbitmaps_offset, NULL,
 			     &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0 )
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
 
-
     /* --------------------- */
     /* Close open datasets   */
     /* --------------------- */
@@ -1152,6 +1184,7 @@ H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
 			 bitmaps_filespace_id, dxpl_id, fastbit->bitmaps))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
 
+
     if (sequence_id == -1) {	/* Disabled for now */
       key_array_size = gatherInfo[mpi_rank].nkeys * sizeof(double);
       sprintf(log_entry, "%d-keys-save-%s", sequence_id, fastbit->column_name);
@@ -1300,9 +1333,10 @@ H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf, size_t *buf_s
 {
     /* The implementation records 3 values (for the anonymous datasets)
      *  {keys_id, offsets_id, bitmaps_id}
+     * For parallel indexing/querying, we add an additional dataset (index_info_group).
      */
     int extra_info = 0;
-    size_t metadata_size = 3 * sizeof(haddr_t);
+    size_t metadata_size = 3 * sizeof(int64_t);
     herr_t ret_value = SUCCEED; /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -1310,7 +1344,7 @@ H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf, size_t *buf_s
 #ifdef H5_HAVE_PARALLEL
     if (H5Xparallel_queries_enabled() > 0) {
         extra_info = 1;
-        metadata_size += sizeof(haddr_t);
+        metadata_size += sizeof(int64_t);
       }
 #endif
     if (buf) {
@@ -1323,7 +1357,7 @@ H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf, size_t *buf_s
 	    if ((fastbit->index_info_group_id != (hid_t)FAIL) && (FAIL == H5Oget_info(fastbit->index_info_group_id, &dset_info)))
                 HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get info for anonymous dataset");
             HDmemcpy(buf_ptr, &dset_info.addr, sizeof(haddr_t));
-            buf_ptr += sizeof(haddr_t);           
+            buf_ptr += sizeof(haddr_t);
 	}
 
         /* Encode keys info */
@@ -1437,179 +1471,201 @@ H5X__fastbit_reconstruct_parallel_index(H5X_fastbit_t *fastbit)
     size_t key_array_size, offset_array_size, bitmap_array_size;
     FastBitDataType fastbit_type;
     hid_t type_id = FAIL, space_id = FAIL;
-    size_t nelmts;
+    size_t nelmts, memelmts;
 
     herr_t ret_value = SUCCEED; /* Return value */
-    int64_t nkeys_totalsize = 0, nkeys_offset = 0;
-    int64_t noffsets_totalsize = 0, noffsets_offset = 0;
-    int64_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
+    hsize_t nkeys_totalsize = 0, nkeys_offset = 0;
+    hsize_t noffsets_totalsize = 0, noffsets_offset = 0;
+    hsize_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
 
     struct fastbit_meta_collection *gatherInfo = NULL;
     hid_t info_space_id = FAIL;
     size_t info_array_size;
-    int i, nranks;
+    size_t i, nranks;
     int mpi_size = H5Xparallel_size();
     int mpi_rank = H5Xparallel_rank();
 
     FUNC_ENTER_NOAPI_NOINIT
 
-    /* Create a transfer property to utilize MPI-IO */
-    if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 )
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't define dataset_xfer property");
-    if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 )
-       HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't set dataset_xfer to MPIO_COLLECTIVE");
-
-    /* Read the spaceinfo from the anonymous dataset.
-     * We will compare the computed total size per private store size
-     * to ensure that things are ok to proceed.
-     */
-    if (FAIL == (info_space_id = H5Dget_space(fastbit->index_info_group_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    /* Note that the npoints is NOT referenced as the number of bytes...
-     * It refers to the number of elements that the dataset was written with.
-     * In our case, we wrote the data as H5T_NATIVE_LLONG, so we'll get the number
-     * of bytes for our calculation of nranks.
-     */
-    if (0 == (info_array_size = (size_t) H5Sget_select_npoints(info_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-    if (NULL == (gatherInfo = (struct fastbit_meta_collection *) H5MM_malloc(info_array_size * sizeof(int64_t))))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate gatherInfo");
-    if (FAIL == H5Dread(fastbit->index_info_group_id, H5T_NATIVE_LLONG,
-                        H5S_ALL, info_space_id, dxpl_id, gatherInfo))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read gatherInfo data");
-
-    /* The gatherInfo array is N groups of 3 int64_t values
-     * It should be a simple thing to divide the total dataset size by
-     * the size of a single entry to compute the number of rank entries
-     * stored.  This needs to match the current MPI size.
-     */
-    nranks = (info_array_size * sizeof(int64_t)) / sizeof(struct fastbit_meta_collection);
-
-    for(i=0; i< nranks; i++) {
-        if (i == mpi_rank) {
-            nkeys_offset = nkeys_totalsize;
-            noffsets_offset = noffsets_totalsize;
-            nbitmaps_offset = nbitmaps_totalsize;
+    if ((fastbit->keys == NULL) ||
+	(fastbit->offsets == NULL) ||
+	(fastbit->bitmaps == NULL)) {
+
+
+        /* Create a transfer property to utilize MPI-IO */
+        if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 )
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't define dataset_xfer property");
+        if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 )
+           HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't set dataset_xfer to MPIO_COLLECTIVE");
+
+        /* Read the spaceinfo from the anonymous dataset.
+         * We will compare the computed total size per private store size
+         * to ensure that things are ok to proceed.
+         */
+        if (FAIL == (info_space_id = H5Dget_space(fastbit->index_info_group_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        /* Note that the npoints is NOT referenced as the number of bytes...
+         * It refers to the number of elements that the dataset was written with.
+         * In our case, we wrote the data as H5T_NATIVE_LLONG, so we'll get the number
+         * of bytes for our calculation of nranks.
+         */
+        if (0 == (info_array_size = (size_t) H5Sget_select_npoints(info_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+        if (NULL == (gatherInfo = (struct fastbit_meta_collection *) H5MM_malloc(info_array_size * sizeof(int64_t))))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate gatherInfo");
+        if (FAIL == H5Dread(fastbit->index_info_group_id, H5T_NATIVE_LLONG,
+                            H5S_ALL, info_space_id, dxpl_id, gatherInfo))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read gatherInfo data");
+
+        /* The gatherInfo array is N groups of 3 int64_t values
+         * It should be a simple thing to divide the total dataset size by
+         * the size of a single entry to compute the number of rank entries
+         * stored.  This needs to match the current MPI size.
+         */
+        nranks = (info_array_size * sizeof(int64_t)) / sizeof(struct fastbit_meta_collection);
+
+        for(i=0; i< nranks; i++) {
+            if (i == mpi_rank) {
+                nkeys_offset = nkeys_totalsize;
+                noffsets_offset = noffsets_totalsize;
+                nbitmaps_offset = nbitmaps_totalsize;
+                fastbit->nelmts = (int64_t)(gatherInfo[i].data_length);
+            }
+            nkeys_totalsize += gatherInfo[i].nkeys +1;
+            noffsets_totalsize += gatherInfo[i].noffsets +1;
+            nbitmaps_totalsize += gatherInfo[i].nbitmaps +1;
         }
-        nkeys_totalsize += gatherInfo[i].nkeys;
-        noffsets_totalsize += gatherInfo[i].noffsets;
-        nbitmaps_totalsize += gatherInfo[i].nbitmaps;
-    }
 
-    /* Prepare to read nkeys */
-    if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+	fastbit->nblocks = mpi_size;
+	fastbit->idx_multiblock = TRUE;
+        fastbit->nelmts = gatherInfo[mpi_rank].data_length;
 
-    HDassert(nkeys_totalsize == key_array_size);
-    fastbit->nkeys = gatherInfo[mpi_rank].nkeys;
-
-    /* Prepare to read noffsets */
-    if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
-    HDassert(noffsets_totalsize == offset_array_size);
-    fastbit->noffsets = gatherInfo[mpi_rank].noffsets;
-
-    /* Prepare to read nbitmaps */
-    if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
-    HDassert(nbitmaps_totalsize == bitmap_array_size);
-    fastbit->nbitmaps = gatherInfo[mpi_rank].nbitmaps;
-
-    if (NULL == (fastbit->keys = (double *) H5MM_malloc(gatherInfo[mpi_rank].nkeys * sizeof(double))))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
-    if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(gatherInfo[mpi_rank].noffsets * sizeof(int64_t))))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-    if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t))))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-
-    /* define the hyperslab selection(s) */
-    if((keys_filespace_id = H5Screate_simple(1, &nkeys_totalsize, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
-    if((offsets_filespace_id = H5Screate_simple(1, &noffsets_totalsize, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
-    if((bitmaps_filespace_id = H5Screate_simple(1, &nbitmaps_totalsize, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
-    
-    /* Memspace descriptors */
-    if((keys_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nkeys, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
-    if((offsets_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].noffsets, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
-    if((bitmaps_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0)
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
-
-    if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, &nkeys_offset, NULL, &gatherInfo[mpi_rank].nkeys, NULL)) < 0 )
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-    if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, &noffsets_offset, NULL, &gatherInfo[mpi_rank].noffsets, NULL)) < 0 )
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-    if ((H5Sselect_hyperslab(bitmaps_filespace_id, H5S_SELECT_SET, &nbitmaps_offset, NULL, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0 )
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-
-    /* Read keys */
-    if (FAIL == H5Dread(fastbit->keys_id, H5T_NATIVE_DOUBLE, keys_memspace_id,
-			keys_filespace_id, dxpl_id, fastbit->keys))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index keys metadata");
+        /* Prepare to read nkeys */
+        if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
 
-    /* Read offsets */
-    if (FAIL == H5Dread(fastbit->offsets_id, H5T_NATIVE_LLONG, offsets_memspace_id,
-			offsets_filespace_id, dxpl_id, fastbit->offsets))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index offsets metadata");
+        HDassert(nkeys_totalsize == key_array_size);
+        fastbit->nkeys = gatherInfo[mpi_rank].nkeys;
 
-    /* Read bitmaps */
-    if (FAIL == H5Dread(fastbit->bitmaps_id, H5T_NATIVE_UINT, bitmaps_memspace_id,
-			bitmaps_filespace_id, dxpl_id, fastbit->bitmaps))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index bitmaps metadata");
+        /* Prepare to read noffsets */
+        if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
 
-    if (sequence_id == -1) {	/* Disabled for now */
-      key_array_size = gatherInfo[mpi_rank].nkeys * sizeof(double);
-      sprintf(log_entry, "%d-keys-read-%s", sequence_id, fastbit->column_name);
-      dump_as_bytes(log_entry, key_array_size, fastbit->keys);
+        HDassert(noffsets_totalsize == offset_array_size);
+        fastbit->noffsets = gatherInfo[mpi_rank].noffsets;
 
-      offset_array_size = gatherInfo[mpi_rank].noffsets * sizeof(int64_t);
-      sprintf(log_entry, "%d-offsets-read-%s", sequence_id, fastbit->column_name);
-      dump_as_bytes(log_entry, offset_array_size, fastbit->offsets);
+        /* Prepare to read nbitmaps */
+        if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
 
-      bitmap_array_size = gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t);
-      sprintf(log_entry, "%d-bitmaps-read-%s", sequence_id,  fastbit->column_name);
-      dump_as_bytes(log_entry, bitmap_array_size, fastbit->bitmaps);
+        HDassert(nbitmaps_totalsize == bitmap_array_size);
+        fastbit->nbitmaps = gatherInfo[mpi_rank].nbitmaps;
 
-      sequence_id++;
-    }
+        if (NULL == (fastbit->keys = (double *) H5MM_malloc(gatherInfo[mpi_rank].nkeys * sizeof(double))))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
+        if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(gatherInfo[mpi_rank].noffsets * sizeof(int64_t))))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
+        if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t))))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
 
+        /* define the hyperslab selection(s) */
+        if((keys_filespace_id = H5Screate_simple(1, &nkeys_totalsize, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
+        if((offsets_filespace_id = H5Screate_simple(1, &noffsets_totalsize, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
+        if((bitmaps_filespace_id = H5Screate_simple(1, &nbitmaps_totalsize, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
+    
+        /* Memspace descriptors */
+        if((keys_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nkeys, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
+        if((offsets_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].noffsets, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
+        if((bitmaps_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");      
+
+        if (H5Sselect_all(keys_memspace_id) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
+        if (H5Sselect_all(offsets_memspace_id) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
+        if (H5Sselect_all(bitmaps_memspace_id) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
+
+
+        if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, &nkeys_offset, NULL, &gatherInfo[mpi_rank].nkeys, NULL)) < 0 )
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
+        if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, &noffsets_offset, NULL, &gatherInfo[mpi_rank].noffsets, NULL)) < 0 )
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
+        if ((H5Sselect_hyperslab(bitmaps_filespace_id, H5S_SELECT_SET, &nbitmaps_offset, NULL, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0 )
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
 
-    /* Reconstruct index */
-    H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
-            "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
+        /* Read keys */
+        if (FAIL == H5Dread(fastbit->keys_id, H5T_NATIVE_DOUBLE, keys_memspace_id,
+    			keys_filespace_id, dxpl_id, fastbit->keys))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index keys metadata");
+
+        /* Read offsets */
+        if (FAIL == H5Dread(fastbit->offsets_id, H5T_NATIVE_LLONG, offsets_memspace_id,
+    			offsets_filespace_id, dxpl_id, fastbit->offsets))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index offsets metadata");
+
+        /* Read bitmaps */
+        if (FAIL == H5Dread(fastbit->bitmaps_id, H5T_NATIVE_UINT, bitmaps_memspace_id,
+    			bitmaps_filespace_id, dxpl_id, fastbit->bitmaps))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index bitmaps metadata");
+
+        if (sequence_id == -1) {	/* Disabled for now */
+          key_array_size = gatherInfo[mpi_rank].nkeys * sizeof(double);
+          sprintf(log_entry, "%d-keys-read-%s", sequence_id, fastbit->column_name);
+          dump_as_bytes(log_entry, key_array_size, fastbit->keys);
+
+          offset_array_size = gatherInfo[mpi_rank].noffsets * sizeof(int64_t);
+          sprintf(log_entry, "%d-offsets-read-%s", sequence_id, fastbit->column_name);
+          dump_as_bytes(log_entry, offset_array_size, fastbit->offsets);
+    
+          bitmap_array_size = gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t);
+          sprintf(log_entry, "%d-bitmaps-read-%s", sequence_id,  fastbit->column_name);
+          dump_as_bytes(log_entry, bitmap_array_size, fastbit->bitmaps);
 
-    /* Get space info */
-    if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
-        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
-    if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
-    if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+          sequence_id++;
+        }
 
-    /* Convert type to FastBit type */
-    if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
 
-    /* Register array */
-    if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
-            fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
-            fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
-            H5X__fastbit_read_bitmaps))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
+        /* Reconstruct index */
+        H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
+                "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
 
-    H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
-    fastbit->idx_reconstructed = TRUE;
+        /* Get space info */
+        if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
+            HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
+        if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+
+        /* We've restored the number of elements in the gatherInfo array (.data_length)
+         * and cached it for use here and possibly elsewhere.
+         */
+        nelmts = (size_t)fastbit->nelmts;
+
+        /* Convert type to FastBit type */
+        if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
+
+        /* Register array */
+        if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
+                fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
+                fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
+                H5X__fastbit_read_bitmaps))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
+
+        H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
+        fastbit->idx_reconstructed = TRUE;
+    }
 
 done:
 
@@ -1656,81 +1712,83 @@ H5X__fastbit_reconstruct_index(H5X_fastbit_t *fastbit)
 
 #ifdef H5_HAVE_PARALLEL
     if (H5Xparallel_queries_enabled() > 0)
+    // if (fastbit->idx_multiblock)
         return H5X__fastbit_reconstruct_parallel_index(fastbit);
 #endif
     FUNC_ENTER_NOAPI_NOINIT
 
     /* TODO don't read keys and offsets if already present */
-    HDassert(!fastbit->keys);
-    HDassert(!fastbit->offsets);
-
-    if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-    fastbit->nkeys = key_array_size / sizeof(double);
-
-    if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-    fastbit->noffsets = offset_array_size / sizeof(int64_t);
+    if ((fastbit->keys == NULL) ||
+	(fastbit->offsets == NULL) ||
+	(fastbit->bitmaps == NULL)) {
 
-    if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-    if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-    fastbit->nbitmaps = bitmap_array_size / sizeof(uint32_t);
-
-    /* allocate buffer to hold data */
-    if (NULL == (fastbit->keys = (double *) H5MM_malloc(key_array_size)))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
-    if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(offset_array_size)))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-    if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(bitmap_array_size)))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-
-    /* Read FastBit keys */
-    if (FAIL == H5Dread(fastbit->keys_id, fastbit->opaque_type_id,
-            H5S_ALL, keys_space_id, H5P_DEFAULT, fastbit->keys))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+        if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+        fastbit->nkeys = key_array_size / sizeof(double);
 
-    /* Read FastBit offsets */
-    if (FAIL == H5Dread(fastbit->offsets_id, fastbit->opaque_type_id,
-            H5S_ALL, offsets_space_id, H5P_DEFAULT, fastbit->offsets))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+        if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+        fastbit->noffsets = offset_array_size / sizeof(int64_t);
 
-    /* Read FastBit bitmaps */
-    if (FAIL == H5Dread(fastbit->bitmaps_id, fastbit->opaque_type_id,
-            H5S_ALL, bitmaps_space_id, H5P_DEFAULT, fastbit->bitmaps))
-        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+        if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+        fastbit->nbitmaps = bitmap_array_size / sizeof(uint32_t);
+
+        /* allocate buffer to hold data */
+        if (NULL == (fastbit->keys = (double *) H5MM_malloc(key_array_size)))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
+        if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(offset_array_size)))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
+        if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(bitmap_array_size)))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
+
+        /* Read FastBit keys */
+        if (FAIL == H5Dread(fastbit->keys_id, fastbit->opaque_type_id,
+                H5S_ALL, keys_space_id, H5P_DEFAULT, fastbit->keys))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+
+        /* Read FastBit offsets */
+        if (FAIL == H5Dread(fastbit->offsets_id, fastbit->opaque_type_id,
+                H5S_ALL, offsets_space_id, H5P_DEFAULT, fastbit->offsets))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+
+        /* Read FastBit bitmaps */
+        if (FAIL == H5Dread(fastbit->bitmaps_id, fastbit->opaque_type_id,
+                H5S_ALL, bitmaps_space_id, H5P_DEFAULT, fastbit->bitmaps))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+
+        /* Reconstruct index */
+        H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
+                "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
 
-    /* Reconstruct index */
-    H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
-            "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
+        /* Get space info */
+        if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
+            HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
+        if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+        if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
 
-    /* Get space info */
-    if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
-        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
-    if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
-    if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+        /* Convert type to FastBit type */
+        if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
 
-    /* Convert type to FastBit type */
-    if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
-
-    /* Register array */
-    if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
-            fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
-            fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
-            H5X__fastbit_read_bitmaps))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
-
-    H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
-    fastbit->idx_reconstructed = TRUE;
+        /* Register array */
+        if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
+                fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
+                fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
+                H5X__fastbit_read_bitmaps))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
 
+        H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
+        fastbit->idx_reconstructed = TRUE;
+    }
 done:
     if (FAIL != type_id)
         H5Tclose(type_id);
@@ -1844,23 +1902,31 @@ H5X__fastbit_evaluate_query(H5X_fastbit_t *fastbit, hid_t query_id,
     if (0 > (nhits = fastbit_selection_evaluate(selection_handle)))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTCOMPUTE, FAIL, "can't evaluate selection");
 
-    if ((nhits != 0) && (NULL == (hit_coords = (uint64_t *) H5MM_malloc(((size_t) nhits) * sizeof(uint64_t)))))
-        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate hit coordinates");
-
-    if ((nhits != 0) && (0 > fastbit_selection_get_coordinates(selection_handle, hit_coords,(uint64_t) nhits, 0U)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get coordinates");
-
-#ifdef H5X_FASTBIT_DEBUG
-    /* TODO remove debug */
-    {
-        int i;
-        printf(" # %s(): Index read contains following coords: ",  __func__);
-        for (i = 0; i < nhits; i++) {
-            printf("%lu ", hit_coords[i]);
+    if (nhits > 0) {
+        if (NULL == (hit_coords = (uint64_t *) H5MM_malloc(((size_t) nhits) * sizeof(uint64_t))))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate hit coordinates");
+
+        if (0 > fastbit_selection_get_coordinates(selection_handle, hit_coords,(uint64_t) nhits, 0U))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get coordinates");
+
+// #ifdef H5X_FASTBIT_DEBUG
+#if 0
+        /* TODO remove debug */
+        {
+            int k;
+	    int64_t loops = nhits/10;
+            int64_t i;
+            printf(" # %s(): Index read contains following %lld coords: ",  __func__, nhits);
+            for (i=0 ; i < loops; i += 10) {
+		printf("\n");
+                for(k=0; k< 10; k++) {
+                    printf("%lu ", hit_coords[i+k]);
+                }
+	    }
+            printf("\n");
         }
-        printf("\n");
-    }
 #endif
+    }
 
     *coords = hit_coords;
     *ncoords = (uint64_t) nhits;
@@ -1868,7 +1934,7 @@ H5X__fastbit_evaluate_query(H5X_fastbit_t *fastbit, hid_t query_id,
 done:
     if (selection_handle)
         fastbit_selection_free(selection_handle);
-    if (FAIL == ret_value) {
+    if ((FAIL == ret_value) && (hit_coords != NULL)) {
         H5MM_free(hit_coords);
     }
     FUNC_LEAVE_NOAPI(ret_value)
@@ -1893,35 +1959,83 @@ H5X__fastbit_create_selection(H5X_fastbit_t *fastbit, hid_t dataspace_id,
     hsize_t count[H5S_MAX_RANK + 1];
     hsize_t start_coord[H5S_MAX_RANK + 1], end_coord[H5S_MAX_RANK + 1], nelmts;
     herr_t ret_value = SUCCEED; /* Return value */
-    size_t i;
+    size_t i, dataset_rank;
+    double t_start, t_end, t_loop, t_select_bounds, t_select_npoints;
+    double t_hyperslab = 0, temp, t0, t1;
+    double t_min = 0.1, t_max = 0, target = 0.01;
 
     FUNC_ENTER_NOAPI_NOINIT
 
-    /* Initialize count */
-    for (i = 0; i < H5S_MAX_RANK; i++)
-        count[i] = 1;
+    /* Check for non-null selections */
+    if (ncoords > 0) {
+        /* Initialize count */
+        for (i = 0; i < H5S_MAX_RANK; i++)
+            count[i] = 1;
 
-    for (i = 0; i < ncoords; i++) {
-        hsize_t new_coords[H5S_MAX_RANK + 1];
-        const hsize_t point = coords[i];
+#ifdef SHOW_TIME
+        t_start = MPI_Wtime();
+#endif
+        for (i = 0; i < ncoords; i++) {
+            hsize_t new_coords[H5S_MAX_RANK + 1];
+            const hsize_t point = coords[i];
+            /* Convert coordinates */
+            if (FAIL == H5VM_array_calc_pre(point, fastbit->dataset_ndims, fastbit->dataset_down_dims, new_coords))
+                HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate coord array");
+
+#ifdef SHOW_TIME
+	    t0 = MPI_Wtime();
+#endif
+            /* Add converted coordinate to selection */
+            if (H5Sselect_hyperslab(dataspace_id, H5S_SELECT_OR, new_coords, NULL, count, NULL))
+                HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to add point to selection");
+#ifdef SHOW_TIME
+	    t1 = MPI_Wtime();
+	    temp = (t1 - t0);
+	    t_hyperslab += temp;
+	    if (t_min == 0) {
+	      t_min = t_max = temp;
+	    }
+	    if (temp < t_min)
+		t_min = temp;
+	    else if (temp > t_max) {
+		t_max = temp;
+		if ((temp > target)) {
+		  printf("exceeded target max (%f) at index = %lld\n", target, i);
+		  target *= 2.0;
+		}
+	    }
+#endif
+        }
 
-        /* Convert coordinates */
-        if (FAIL == H5VM_array_calc_pre(point, fastbit->dataset_ndims,
-                fastbit->dataset_down_dims, new_coords))
-            HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate coord array");
+#ifdef SHOW_TIME
+        t_end = MPI_Wtime();
+	t_loop = (t_end - t_start);
+	t_start = t_end;
+#endif
 
-        /* Add converted coordinate to selection */
-        if (H5Sselect_hyperslab(dataspace_id, H5S_SELECT_OR, new_coords, NULL, count, NULL))
-            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to add point to selection");
+        if (FAIL == H5Sget_select_bounds(dataspace_id, start_coord, end_coord))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "unable to get bounds");
+#ifdef SHOW_TIME
+	t_end = MPI_Wtime();
+	t_select_bounds = (t_end - t_start);
+	t_start = t_end;
+#endif
+        if (0 == (nelmts = (hsize_t) H5Sget_select_npoints(dataspace_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+#ifdef SHOW_TIME
+	t_end = MPI_Wtime();	
+	t_select_npoints = (t_end - t_start);
+
+	printf("\tt_loop = %f (cummulative)\n", t_loop);
+	printf("\tt_hyperslab = %f (cummulative)\n", t_hyperslab);
+	printf("\tt_select_bounds = %f\n", t_select_bounds);
+	printf("\tt_select_npoints = %f\n", t_select_npoints);
+	printf("\tt_min hyperslab = %f\n", t_min);
+	printf("\tt_max hyperslab = %f\n", t_max);
+#endif
+        H5X_FASTBIT_LOG_DEBUG("Created dataspace from index with %llu elements [(%llu, %llu):(%llu, %llu)]",
+                nelmts, start_coord[0], start_coord[1], end_coord[0], end_coord[1]);
     }
-
-    if (FAIL == H5Sget_select_bounds(dataspace_id, start_coord, end_coord))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "unable to get bounds");
-    if (0 == (nelmts = (hsize_t) H5Sget_select_npoints(dataspace_id)))
-        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-    H5X_FASTBIT_LOG_DEBUG("Created dataspace from index with %llu elements [(%llu, %llu):(%llu, %llu)]",
-            nelmts, start_coord[0], start_coord[1], end_coord[0], end_coord[1]);
-
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 }
@@ -1946,6 +2060,7 @@ H5X_fastbit_create(hid_t dataset_id, hid_t xcpl_id, hid_t H5_ATTR_UNUSED xapl_id
     void *buf = NULL;
     size_t buf_size;
     hbool_t read_on_create = TRUE;
+    double t_start, t_end;
 
     FUNC_ENTER_NOAPI_NOINIT
     H5X_FASTBIT_LOG_DEBUG("Enter");
@@ -1962,17 +2077,27 @@ H5X_fastbit_create(hid_t dataset_id, hid_t xcpl_id, hid_t H5_ATTR_UNUSED xapl_id
 
     if (read_on_create) {
         /* Get data from dataset */
+#ifdef SHOW_TIME
+        t_start = MPI_Wtime();        
+#endif
         if (FAIL == H5X__fastbit_read_data(dataset_id, fastbit->column_name, &buf, &buf_size))
             HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get data from dataset");
-
+#ifdef SHOW_TIME
+	t_end = MPI_Wtime();
+	printf("H5X__fastbit_read_data finished reading %lld bytes in %f seconds\n", buf_size, t_end -t_start);
         /* Index data */
+	t_start = t_end;
+#endif
         if (FAIL == H5X__fastbit_build_index(fastbit))
            HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create index data from dataset");
+#ifdef SHOW_TIME
+	t_end = MPI_Wtime();
+	printf("H5X__fastbit_build_index completed in %f seconds\n", t_end -t_start);
+#endif
     }
 
     /* Serialize metadata for H5X interface */
-    if (FAIL == H5X__fastbit_serialize_metadata(fastbit, NULL,
-            &private_metadata_size))
+    if (FAIL == H5X__fastbit_serialize_metadata(fastbit, NULL, &private_metadata_size))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get plugin metadata size");
 
     if (NULL == (fastbit->private_metadata = H5MM_malloc(private_metadata_size)))
@@ -2015,7 +2140,6 @@ H5X_fastbit_remove(hid_t file_id, size_t metadata_size, void *metadata)
 
     FUNC_ENTER_NOAPI_NOINIT
     H5X_FASTBIT_LOG_DEBUG("Enter");
-
     if (metadata_size < (3 * sizeof(haddr_t)))
         HGOTO_ERROR(H5E_INDEX, H5E_BADVALUE, FAIL, "metadata size is not valid");
 
@@ -2174,7 +2298,6 @@ H5X_fastbit_post_update(void *idx_handle, const void *data, hid_t dataspace_id,
     FUNC_ENTER_NOAPI_NOINIT
 
     H5X_FASTBIT_LOG_DEBUG("Calling H5X_fastbit_post_update");
-
     if (NULL == fastbit)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL index handle");
 
@@ -2215,6 +2338,7 @@ H5X_fastbit_query(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id,
     uint64_t *coords = NULL;
     uint64_t ncoords = 0;
     hid_t ret_value = FAIL; /* Return value */
+    double t_start, t_end;
 
     FUNC_ENTER_NOAPI_NOINIT
     H5X_FASTBIT_LOG_DEBUG("Enter");
@@ -2234,18 +2358,31 @@ H5X_fastbit_query(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id,
     if (FAIL == H5Sselect_none(ret_space_id))
         HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "can't reset selection");
 
+#ifdef SHOW_TIME
+    t_start = MPI_Wtime();
+#endif
+
     /* Get matching coordinates values from query */
     if (FAIL == H5X__fastbit_evaluate_query(fastbit, query_id, &coords, &ncoords))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get query range");
-
+#ifdef SHOW_TIME
+    t_end = MPI_Wtime();
+    printf("H5X__fastbit_evaluate_query took %f seconds to process %lld of %lld total elements, resulting in %lld coords\n", t_end - t_start, fastbit->nelmts, fastbit->dataset_dims[0], ncoords);
+    t_start = t_end;
+#endif
     /* Create selection */
     if (FAIL == H5X__fastbit_create_selection(fastbit, ret_space_id, coords, ncoords))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get query range");
 
+#ifdef SHOW_TIME
+    t_end = MPI_Wtime();
+    printf("H5X__fastbit_create_selection took %f seconds\n", t_end - t_start);
+#endif
     ret_value = ret_space_id;
 
 done:
-    H5MM_free(coords);
+    if (coords != NULL)
+        H5MM_free(coords);
     if (FAIL != space_id)
         H5Sclose(space_id);
     if ((FAIL == ret_value) && (FAIL != ret_space_id))
diff --git a/src/H5Xpublic.h b/src/H5Xpublic.h
index 88fa008..4cbdaf7 100644
--- a/src/H5Xpublic.h
+++ b/src/H5Xpublic.h
@@ -152,7 +152,10 @@ H5_DLL int H5Xparallel_queries_enabled(void);
 H5_DLL int H5Xslab_set(hid_t filespace_id, hsize_t **start, hsize_t **count, hsize_t **stride, hsize_t **block);
 H5_DLL int H5Xparallel_rank(void);
 H5_DLL int H5Xparallel_size(void);
+H5_DLL unsigned H5Xallreduce_unsigned_status(unsigned status, hbool_t do_summation);
+H5_DLL int H5Xallreduce_int_status(int status, hbool_t do_summation);
 H5_DLL herr_t H5Xallgather_by_size(void *xdata, int nelems, int typesize);
+
 #endif
 
 #ifdef __cplusplus