[svn-r6527] Purpose:

    Bug Fix & Update

Description:
    FPHDF5 was creating files which didn't have the EOA field in the
    superblock set correctly. It turns out that the SAP was keeping this
    information to itself instead of giving it to the client processes.

    Naughty SAP!

Solution:
    Have the SAP send this information back to the clients so that they
    can update the superblock as necessary. This now creates a file (with
    just the root group) that looks correct! Only problem is that there's
    extra file space being allocated. Also, at program termination,
    there's an infinite loop...

Platforms tested:
    H5committests (run by hand on burrwhite, arabica, and modi4)

Misc. update:

7 files changed, 58 insertions, 107 deletions

diff --git a/src/H5F.c b/src/H5F.c
index 464c2a8..a8ba371 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -1828,7 +1828,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d
 	    file = NULL; /*to prevent destruction of wrong file*/
 	    HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for read-only");
 	}
-	file = H5F_new(file->shared, fcpl_id, fapl_id);
+
+        if ((file = H5F_new(file->shared, fcpl_id, fapl_id)) == NULL)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create new file object");
+
 	lf = file->shared->lf;
     } else if (flags!=tent_flags) {
 	/*
@@ -2672,42 +2675,23 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, H5F_scope_t scope, unsigned flags)
     if (flags & H5F_FLUSH_ALLOC_ONLY) {
 	haddr_t addr;
 
-#ifdef H5_HAVE_FPHDF5
         /*
-         * If this is an FPHDF5 file driver, then we only want the
-         * captain process allocating the space. The rest of the
-         * processes should just get the broadcast message sent from the
-         * captain.
+         * Allocate space for the userblock, superblock, and driver info
+         * block. We do it with one allocation request because the
+         * userblock and superblock need to be at the beginning of the
+         * file and only the first allocation request is required to
+         * return memory at format address zero.
+         *
+         * Note: This is safe for FPHDF5. We only set H5F_FLUSH_ALLOC_ONLY
+         * from the H5F_open function. That function sets it so that only
+         * the captain process will actually perform any allocations,
+         * which is what we want here. In the H5FD_alloc function, the
+         * allocated address is broadcast to the other processes.
          */
-        if (!H5FD_is_fphdf5_driver(f->shared->lf) ||
-                H5FD_fphdf5_is_captain(f->shared->lf)) {
-#endif  /* H5_HAVE_FPHDF5 */
-
-            /*
-             * Allocate space for the userblock, superblock, and driver
-             * info block. We do it with one allocation request because
-             * the userblock and superblock need to be at the beginning
-             * of the file and only the first allocation request is
-             * required to return memory at format address zero.
-             */
-            H5_CHECK_OVERFLOW(f->shared->base_addr,haddr_t,hsize_t);
-
-            addr = H5FD_alloc(f->shared->lf, H5FD_MEM_SUPER, dxpl_id,
-                              ((hsize_t)f->shared->base_addr +
-                                            superblock_size + driver_size));
-
-#ifdef H5_HAVE_FPHDF5
-        }
+        H5_CHECK_OVERFLOW(f->shared->base_addr,haddr_t,hsize_t);
 
-        if (H5FD_is_fphdf5_driver(f->shared->lf)) {
-            int mrc;
-
-            if ((mrc = MPI_Bcast(&addr, 1, HADDR_AS_MPI_TYPE,
-                                 (int)H5FP_capt_barrier_rank,
-                                 H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
-                HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mrc);
-        }
-#endif  /* H5_HAVE_FPHDF5 */
+        addr = H5FD_alloc(f->shared->lf, H5FD_MEM_SUPER, dxpl_id,
+                          ((hsize_t)f->shared->base_addr + superblock_size + driver_size));
 
         if (HADDR_UNDEF == addr)
             HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL,
@@ -2763,25 +2747,12 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, H5F_scope_t scope, unsigned flags)
 	} /* end if */
     } /* end else */
 
-#ifdef H5_HAVE_FPHDF5
-    /*
-     * We only want the captain to perform the flush of the metadata to
-     * the file.
-     */
-    if (!H5FD_is_fphdf5_driver(f->shared->lf) ||
-            H5FD_fphdf5_is_captain(f->shared->lf)) {
-#endif  /* H5_HAVE_FPHDF5 */
-
-        /* If we're not just allocating... */
-        if ((flags & H5F_FLUSH_ALLOC_ONLY) == 0)
-            /* ...flush file buffers to disk. */
-            if (H5FD_flush(f->shared->lf, dxpl_id,
-                           (unsigned)((flags & H5F_FLUSH_CLOSING) > 0)) < 0)
-                HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "low level flush failed");
-
-#ifdef H5_HAVE_FPHDF5
-    }
-#endif  /* H5_HAVE_FPHDF5 */
+    /* If we're not just allocating... */
+    if ((flags & H5F_FLUSH_ALLOC_ONLY) == 0)
+        /* ...flush file buffers to disk. */
+        if (H5FD_flush(f->shared->lf, dxpl_id,
+                       (unsigned)((flags & H5F_FLUSH_CLOSING) > 0)) < 0)
+            HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "low level flush failed");
 
     /* Check flush errors for children - errors are already on the stack */
     ret_value = (nerrors ? FAIL : SUCCEED);
diff --git a/src/H5FD.c b/src/H5FD.c
index a183b48..a2bf906 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -1319,6 +1319,7 @@ H5FD_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
         unsigned        capt_only = 0;
         H5FP_status_t   status;
         H5P_genplist_t *plist;
+        H5FP_alloc_t    fp_alloc;
 
         /* Get the data xfer property list */
         if ((plist = H5I_object(dxpl_id)) == NULL)
@@ -1333,25 +1334,30 @@ H5FD_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
          * If the captain is the only one who should allocate resources,
          * then do just that...
          */
-        if (!capt_only || H5FD_fphdf5_is_captain(file))
+        if (!capt_only || H5FD_fphdf5_is_captain(file)) {
             /* Send the request to the SAP */
-            if (H5FP_request_allocate(file, type, size, &ret_value, &req_id,
-                                      &status) != SUCCEED)
+            if (H5FP_request_allocate(file, type, size, &fp_alloc.addr,
+                                      &fp_alloc.eoa, &req_id, &status) != SUCCEED)
                 /* FIXME: Should we check the "status" variable here? */
                 HGOTO_ERROR(H5E_FPHDF5, H5E_CANTALLOC, HADDR_UNDEF,
                             "server couldn't allocate from file");
+        }
 
         if (capt_only) {
             int mrc;
 
-            if ((mrc = MPI_Bcast(&ret_value, 1, HADDR_AS_MPI_TYPE,
+            if ((mrc = MPI_Bcast(&fp_alloc, 1, H5FP_alloc,
                                  (int)H5FP_capt_barrier_rank,
                                  H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
                 HMPI_GOTO_ERROR(HADDR_UNDEF, "MPI_Bcast failed", mrc);
         }
 
+
+        /* Set the EOA for all processes. This doesn't fail. */
+        file->cls->set_eoa(file, fp_alloc.eoa);
+
         /* We've succeeded -- return the value */
-        HGOTO_DONE(ret_value);
+        HGOTO_DONE(fp_alloc.addr);
     }
 #endif  /* H5_HAVE_FPHDF5 */
 
diff --git a/src/H5FDfphdf5.c b/src/H5FDfphdf5.c
index ce293bb..3cbe426 100644
--- a/src/H5FDfphdf5.c
+++ b/src/H5FDfphdf5.c
@@ -1402,9 +1402,6 @@ H5FD_fphdf5_write(H5FD_t *_file, H5FD_mem_t mem_type, hid_t dxpl_id,
 
     FUNC_ENTER_NOAPI(H5FD_fphdf5_write, FAIL);
 
-HDfprintf(stderr, "%s: Entering: rank==%d, addr==%a, size==%Zu\n",
-          FUNC, file->mpi_rank, addr, size);
-
     /* check args */
     assert(file);
     assert(file->pub.driver_id == H5FD_FPHDF5);
@@ -1447,9 +1444,6 @@ HDfprintf(stderr, "%s: Entering: rank==%d, addr==%a, size==%Zu\n",
                         &block_before_meta_write) < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get H5AC property");
 
-HDfprintf(stderr, "%s: %d: block_before_meta_write == %d\n", FUNC,
-          H5FD_fphdf5_mpi_rank(_file), block_before_meta_write);
-
         if (block_before_meta_write)
             if ((mrc = MPI_Barrier(file->barrier_comm)) != MPI_SUCCESS)
                 HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mrc);
@@ -1478,8 +1472,6 @@ HDfprintf(stderr, "%s:%d: Couldn't write metadata to SAP (%d)\n",
 
         switch (sap_status) {
         case H5FP_STATUS_OK:
-HDfprintf(stderr, "%s: %d: Wrote the data to the SAP\n", FUNC,
-          H5FD_fphdf5_mpi_rank(_file));
             /* WAH-HOO! We've written it! We can leave now */
             /* Forget the EOF value (see H5FD_fphdf5_get_eof()) */
             file->eof = HADDR_UNDEF;
@@ -1501,7 +1493,6 @@ HDfprintf(stderr, "%s: Couldn't write metadata to SAP (%d)\n",
     ret_value = H5FD_fphdf5_write_real(_file, dxpl_id, mpi_off, size_i, buf);
 
 done:
-HDfprintf(stderr, "%s: Leaving\n", FUNC);
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
@@ -1536,9 +1527,6 @@ H5FD_fphdf5_write_real(H5FD_t *_file, hid_t dxpl_id, MPI_Offset mpi_off, int siz
 
     FUNC_ENTER_NOAPI(H5FD_fphdf5_write_real, FAIL);
 
-HDfprintf(stderr, "%s: %d: Entering: addr=%a, size=%Zu\n", FUNC,
-          H5FD_fphdf5_mpi_rank(_file), (haddr_t)mpi_off, size);
-
     /* check args */
     assert(file);
     assert(file->pub.driver_id == H5FD_FPHDF5);
@@ -1603,22 +1591,6 @@ HDfprintf(stderr, "%s: %d: Entering: addr=%a, size=%Zu\n", FUNC,
     /* Write the data. */
     assert(xfer_mode == H5FD_MPIO_INDEPENDENT || xfer_mode == H5FD_MPIO_COLLECTIVE);
 
-    {
-        int i;
-
-        sleep(3);
-        HDfprintf(stderr, "%s: writing at %a\n", FUNC, (haddr_t)mpi_off);
-
-        for (i = 0; i < size; ++i) {
-            if (i % 7 == 0)
-                HDfprintf(stderr, "\n");
-
-            HDfprintf(stderr, "\t0x%02x", 0xff & ((char*)buf)[i]);
-        }
-
-        HDfprintf(stderr, "\n");
-    }
-
     if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
         /*OKAY: CAST DISCARDS CONST QUALIFIER*/
         if ((mrc = MPI_File_write_at(file->f, mpi_off, (void*)buf,
@@ -1688,7 +1660,6 @@ HDfprintf(stderr, "%s: %d: Entering: addr=%a, size=%Zu\n", FUNC,
     file->eof = HADDR_UNDEF;
 
 done:
-HDfprintf(stderr, "%s: %d: Leaving (%d)\n", FUNC, H5FD_fphdf5_mpi_rank(_file), ret_value);
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
@@ -1715,9 +1686,6 @@ H5FD_fphdf5_flush(H5FD_t *_file, hid_t dxpl_id, unsigned closing)
 
     FUNC_ENTER_NOAPI(H5FD_fphdf5_flush, FAIL);
 
-HDfprintf(stderr, "%s: %d: Entering\n",
-          FUNC, H5FD_fphdf5_mpi_rank(_file));
-
     /* check args */
     assert(file);
     assert(file->pub.driver_id == H5FD_FPHDF5);
@@ -1745,29 +1713,28 @@ HDfprintf(stderr, "%s: %d: Entering\n",
          * finished making it the shorter length, potentially truncating
          * the file and dropping the new data written)
          */
-HDfprintf(stderr, "%s: %d: MPI_Barrier==%d\n", FUNC, file->mpi_rank, file->barrier_comm);
-        if ((mrc = MPI_Barrier(file->barrier_comm)) != MPI_SUCCESS)
+        if ((mrc = MPI_Barrier(H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mrc);
 
         /* Update the 'last' eoa value */
         file->last_eoa = file->eoa;
     }
 
-    if (H5FP_request_flush_metadata(_file, file->file_id, dxpl_id,
-                                    &req_id, &status) != SUCCEED) {
-        /* FIXME: This failed */
+    /* Only the captain process needs to flush the metadata. */
+    if (H5FD_fphdf5_is_captain(_file)) {
+        if (H5FP_request_flush_metadata(_file, file->file_id, dxpl_id,
+                                        &req_id, &status) != SUCCEED) {
+            /* FIXME: This failed */
 HDfprintf(stderr, "%s:%d: Flush failed (%d)\n", FUNC, __LINE__, status);
-    }
+        }
 
-    /* Only sync the file if we are not going to immediately close it */
-    if (!closing)
-        if ((mrc = MPI_File_sync(file->f)) != MPI_SUCCESS)
-            HMPI_GOTO_ERROR(FAIL, "MPI_File_sync failed", mrc);
+        /* Only sync the file if we are not going to immediately close it */
+        if (!closing)
+            if ((mrc = MPI_File_sync(file->f)) != MPI_SUCCESS)
+                HMPI_GOTO_ERROR(FAIL, "MPI_File_sync failed", mrc);
+    }
 
 done:
-HDfprintf(stderr, "%s: %d: Leaving\n",
-          FUNC, H5FD_fphdf5_mpi_rank(_file));
-
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
diff --git a/src/H5FP.c b/src/H5FP.c
index ab92b54..60c73e3 100644
--- a/src/H5FP.c
+++ b/src/H5FP.c
@@ -399,7 +399,7 @@ H5FP_commit_sap_datatypes(void)
 
     /* Commit the H5FP_alloc datatype */
     block_length[0] = 4;
-    block_length[1] = 1;
+    block_length[1] = 2;
     old_types[0] = MPI_UNSIGNED;
     old_types[1] = HADDR_AS_MPI_TYPE;
     MPI_Address(&sap_alloc.req_id, &displs[0]);
diff --git a/src/H5FPclient.c b/src/H5FPclient.c
index 12080fa..4048120 100644
--- a/src/H5FPclient.c
+++ b/src/H5FPclient.c
@@ -613,7 +613,8 @@ done:
  */
 herr_t
 H5FP_request_allocate(H5FD_t *file, H5FD_mem_t mem_type, hsize_t size,
-                      haddr_t *addr, unsigned *req_id, H5FP_status_t *status)
+                      haddr_t *addr, haddr_t *eoa, unsigned *req_id,
+                      H5FP_status_t *status)
 {
     H5FP_alloc_t    sap_alloc;
     H5FP_request_t  req;
@@ -652,6 +653,7 @@ H5FP_request_allocate(H5FD_t *file, H5FD_mem_t mem_type, hsize_t size,
 
     *status = H5FP_STATUS_OK;
     *addr = sap_alloc.addr;
+    *eoa = sap_alloc.eoa;
 
 done:
     *req_id = req.req_id;
diff --git a/src/H5FPprivate.h b/src/H5FPprivate.h
index 101e3a7..ea36397 100644
--- a/src/H5FPprivate.h
+++ b/src/H5FPprivate.h
@@ -231,6 +231,7 @@ typedef struct {
     H5FP_status_t   status;     /* Status of the request                    */
     H5FD_mem_t      mem_type;   /* Type of memory updated, if req'd         */
     haddr_t         addr;       /* Address of the metadata                  */
+    haddr_t         eoa;        /* End of address space                     */
 } H5FP_alloc_t;
 
 extern MPI_Datatype H5FP_alloc; /* MPI datatype for the H5FP_alloc obj      */
@@ -285,7 +286,8 @@ extern herr_t H5FP_request_close(H5FD_t *file, unsigned sap_file_id,
 
 extern herr_t H5FP_request_allocate(H5FD_t *file, H5FD_mem_t mem_type,
                                     hsize_t size, haddr_t *addr,
-                                    unsigned *req_id, H5FP_status_t *status);
+                                    haddr_t *eoa, unsigned *req_id,
+                                    H5FP_status_t *status);
 extern herr_t H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type,
                                 haddr_t addr, hsize_t size,
                                 unsigned *req_id, H5FP_status_t *status);
diff --git a/src/H5FPserver.c b/src/H5FPserver.c
index dc85ad2..01848c7 100644
--- a/src/H5FPserver.c
+++ b/src/H5FPserver.c
@@ -1497,10 +1497,13 @@ H5FP_sap_handle_alloc_request(H5FP_request_t *req)
          * Whatta pain.
          */
         if ((sap_alloc.addr = H5FD_alloc((H5FD_t*)&info->file, req->mem_type, H5P_DEFAULT,
-                                     req->meta_block_size)) == HADDR_UNDEF)
+                                         req->meta_block_size)) == HADDR_UNDEF)
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL,
                         "SAP unable to allocate file memory");
 
+        /* Get the EOA from the file. This call doesn't fail. */
+        sap_alloc.eoa = ((H5FD_t*)&info->file)->cls->get_eoa((H5FD_t*)&info->file);
+
         if ((mrc = MPI_Send(&sap_alloc, 1, H5FP_alloc, (int)req->proc_rank,
                             H5FP_TAG_ALLOC, H5FP_SAP_COMM)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);