[svn-r6556] Purpose:

    Update & Bug Fix

Description:
    The "free" protocol was missing. Added that to the server side.

    When doing a "create" of a file (with no other data structures
    created), the freespace in the file wasn't being reclaimed.

Solution:
    After adding the free protocol, we put the burden of running through
    the FD_free function on the SAP instead of each client.

Platforms tested:
    Linux

Misc. update:

5 files changed, 160 insertions, 72 deletions

diff --git a/src/H5F.c b/src/H5F.c
index a8ba371..8fd0c63 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -2552,33 +2552,56 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, H5F_scope_t scope, unsigned flags)
          * file.
          */
         if (flags & H5F_FLUSH_INVALIDATE) {
-            if (f->shared->lf->feature_flags & H5FD_FEAT_AGGREGATE_METADATA) {
-                /* Return the unused portion of the metadata block to a free list */
-                if (f->shared->lf->eoma != 0)
-                    if (H5FD_free(f->shared->lf, H5FD_MEM_DEFAULT, dxpl_id,
-                                  f->shared->lf->eoma,
-                                  f->shared->lf->cur_meta_block_size) < 0)
-                        HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
-                                    "can't free metadata block");
-
-                /* Reset metadata block information, just in case */
-                f->shared->lf->eoma=0;
-                f->shared->lf->cur_meta_block_size=0;
-            } /* end if */
+#ifdef H5_HAVE_FPHDF5
+            /*
+             * If this is not the SAP, then we want to send a "free"
+             * command to the SAP to free up the EOMA and EOSDA
+             * information. This might also update the EOA information on
+             * the clients...
+             */
+            if (H5FD_is_fphdf5_driver(f->shared->lf) && !H5FD_fphdf5_is_sap(f->shared->lf)) {
+                unsigned        req_id;
+                H5FP_status_t   status;
+
+                /* Send the request to the SAP */
+                if (H5FP_request_free(f->shared->lf, &req_id, &status) != SUCCEED)
+                    /* FIXME: Should we check the "status" variable here? */
+                    HGOTO_ERROR(H5E_FPHDF5, H5E_CANTFREE, FAIL,
+                                "server couldn't free from file");
+            } else {
+#endif  /* H5_HAVE_FPHDF5 */
 
-            if (f->shared->lf->feature_flags & H5FD_FEAT_AGGREGATE_SMALLDATA) {
-                /* Return the unused portion of the "small data" block to a free list */
-                if (f->shared->lf->eosda != 0)
-                    if (H5FD_free(f->shared->lf, H5FD_MEM_DRAW, dxpl_id,
-                                  f->shared->lf->eosda,
-                                  f->shared->lf->cur_sdata_block_size) < 0)
-                        HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
-                                    "can't free 'small data' block");
-
-                /* Reset "small data" block information, just in case */
-                f->shared->lf->eosda=0;
-                f->shared->lf->cur_sdata_block_size=0;
-            } /* end if */
+                if (f->shared->lf->feature_flags & H5FD_FEAT_AGGREGATE_METADATA) {
+                    /* Return the unused portion of the metadata block to a free list */
+                    if (f->shared->lf->eoma != 0)
+                        if (H5FD_free(f->shared->lf, H5FD_MEM_DEFAULT, dxpl_id,
+                                      f->shared->lf->eoma,
+                                      f->shared->lf->cur_meta_block_size) < 0)
+                            HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
+                                        "can't free metadata block");
+
+                    /* Reset metadata block information, just in case */
+                    f->shared->lf->eoma=0;
+                    f->shared->lf->cur_meta_block_size=0;
+                } /* end if */
+
+                if (f->shared->lf->feature_flags & H5FD_FEAT_AGGREGATE_SMALLDATA) {
+                    /* Return the unused portion of the "small data" block to a free list */
+                    if (f->shared->lf->eosda != 0)
+                        if (H5FD_free(f->shared->lf, H5FD_MEM_DRAW, dxpl_id,
+                                      f->shared->lf->eosda,
+                                      f->shared->lf->cur_sdata_block_size) < 0)
+                            HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
+                                        "can't free 'small data' block");
+
+                    /* Reset "small data" block information, just in case */
+                    f->shared->lf->eosda=0;
+                    f->shared->lf->cur_sdata_block_size=0;
+                } /* end if */
+
+#ifdef H5_HAVE_FPHDF5
+            }
+#endif  /* H5_HAVE_FPHDF5 */
         } /* end if */
 
         /* flush the data sieve buffer, if we have a dirty one */
diff --git a/src/H5FD.c b/src/H5FD.c
index a2bf906..36749bb 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -2083,27 +2083,6 @@ H5FD_free(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t si
     assert(file->cls);
     assert(type >= 0 && type < H5FD_MEM_NTYPES);
 
-#ifdef H5_HAVE_FPHDF5
-    /*
-     * When we're using the FPHDF5 driver, allocate from the SAP. If this
-     * is the SAP executing this code, then skip the send to the SAP and
-     * try to do the actual allocations.
-     */
-    if (file->driver_id == H5FD_FPHDF5 && !H5FD_fphdf5_is_sap(file)) {
-        unsigned        req_id;
-        H5FP_status_t   status;
-
-        /* Send the request to the SAP */
-        if (H5FP_request_free(file, type, addr, size, &req_id, &status) != SUCCEED)
-            /* FIXME: Should we check the "status" variable here? */
-            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTFREE, FAIL,
-                        "server couldn't free from file");
-
-        /* We've succeeded -- return the value */
-        HGOTO_DONE(ret_value);
-    }
-#endif  /* H5_HAVE_FPHDF5 */
-
     if (!H5F_addr_defined(addr) || addr>file->maxaddr ||
             H5F_addr_overflow(addr, size) || addr+size>file->maxaddr)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid region");
diff --git a/src/H5FPclient.c b/src/H5FPclient.c
index 4048120..75149e8 100644
--- a/src/H5FPclient.c
+++ b/src/H5FPclient.c
@@ -669,10 +669,9 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type, haddr_t addr,
-                  hsize_t size, unsigned *req_id, H5FP_status_t *status)
+H5FP_request_free(H5FD_t *file, unsigned *req_id, H5FP_status_t *status)
 {
-    H5FP_reply_t    sap_reply;
+    H5FP_alloc_t    sap_alloc;
     H5FP_request_t  req;
     MPI_Status      mpi_status;
     int             mrc;
@@ -687,13 +686,10 @@ H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type, haddr_t addr,
 
     HDmemset(&req, 0, sizeof(req));
 
-    req.req_type = H5FP_REQ_ALLOC;
+    req.req_type = H5FP_REQ_FREE;
     req.req_id = H5FP_gen_request_id();
     req.file_id = H5FD_fphdf5_file_id(file);
     req.proc_rank = H5FD_fphdf5_mpi_rank(file);
-    req.mem_type = mem_type;
-    req.addr = addr;
-    req.meta_block_size = size; /* use this field as the size to free */
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
@@ -701,13 +697,19 @@ H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type, haddr_t addr,
 
     HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-    if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply, (int)H5FP_sap_rank,
-                        H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+    if ((mrc = MPI_Recv(&sap_alloc, 1, H5FP_alloc, (int)H5FP_sap_rank,
+                        H5FP_TAG_ALLOC, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
-    if (sap_reply.status != H5FP_STATUS_OK)
+    if (sap_alloc.status != H5FP_STATUS_OK)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't free space on server");
 
+    if ((mrc = MPI_Bcast(&sap_alloc, 1, H5FP_alloc, (int)H5FP_capt_barrier_rank,
+                         H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mrc);
+
+    /* Set the EOA for all processes. This doesn't fail. */
+    file->cls->set_eoa(file, sap_alloc.eoa);
     *status = H5FP_STATUS_OK;
 
 done:
diff --git a/src/H5FPprivate.h b/src/H5FPprivate.h
index ea36397..5ae04ca 100644
--- a/src/H5FPprivate.h
+++ b/src/H5FPprivate.h
@@ -157,6 +157,10 @@ typedef enum sap_status {
     /* Bad file ID */
     H5FP_STATUS_BAD_FILE_ID,
 
+    /* Can't do memory things */
+    H5FP_STATUS_CANT_ALLOC,
+    H5FP_STATUS_CANT_FREE,
+
     /* Reserved for completely disasterous failures which require an abort */
     H5FP_STATUS_CATASTROPHIC
 } H5FP_status_t;
@@ -283,14 +287,11 @@ extern herr_t H5FP_request_flush_metadata(H5FD_t *file, unsigned file_id,
                                           H5FP_status_t *status);
 extern herr_t H5FP_request_close(H5FD_t *file, unsigned sap_file_id,
                                  unsigned *req_id, H5FP_status_t *status);
-
 extern herr_t H5FP_request_allocate(H5FD_t *file, H5FD_mem_t mem_type,
                                     hsize_t size, haddr_t *addr,
                                     haddr_t *eoa, unsigned *req_id,
                                     H5FP_status_t *status);
-extern herr_t H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type,
-                                haddr_t addr, hsize_t size,
-                                unsigned *req_id, H5FP_status_t *status);
+extern herr_t H5FP_request_free(H5FD_t *file, unsigned *req_id, H5FP_status_t *status);
 
 /* NOTE: Don't use these functions outside of the H5FP* modules! */
 extern herr_t H5FP_send_metadata(const char *mdata, int len, int to);
diff --git a/src/H5FPserver.c b/src/H5FPserver.c
index 01848c7..a46e651 100644
--- a/src/H5FPserver.c
+++ b/src/H5FPserver.c
@@ -161,6 +161,7 @@ static herr_t   H5FP_sap_handle_write_request(H5FP_request_t *req,
 static herr_t   H5FP_sap_handle_flush_request(H5FP_request_t *req);
 static herr_t   H5FP_sap_handle_close_request(H5FP_request_t *req);
 static herr_t   H5FP_sap_handle_alloc_request(H5FP_request_t *req);
+static herr_t   H5FP_sap_handle_free_request(H5FP_request_t *req);
 
 /*
  *===----------------------------------------------------------------------===
@@ -233,6 +234,9 @@ H5FP_sap_receive_loop(void)
         case H5FP_REQ_ALLOC:
             hrc = H5FP_sap_handle_alloc_request(&req);
             break;
+        case H5FP_REQ_FREE:
+            hrc = H5FP_sap_handle_free_request(&req);
+            break;
         case H5FP_REQ_STOP:
             hrc = SUCCEED;
             if (++stop == comm_size - 1)
@@ -1444,13 +1448,17 @@ H5FP_sap_handle_close_request(H5FP_request_t *req)
         if (MPI_Comm_size(H5FP_SAP_COMM, &comm_size) != MPI_SUCCESS)
             HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_size failed");
 
-        if (++info->closing == comm_size - 1)
-            /* all processes have closed the file - remove it from list */
+        if (++info->closing == comm_size - 1) {
+            /* Free the freelist -- this call never fails */
+            H5FD_free_freelist((H5FD_t*)&info->file);
+
+            /* All processes have closed the file - remove it from list */
             if (H5FP_remove_file_id_from_list(req->file_id) != SUCCEED) {
                 exit_state = H5FP_STATUS_BAD_FILE_ID;
                 HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL,
                             "cannot remove file ID from list");
             }
+        }
     }
 
 done:
@@ -1476,12 +1484,12 @@ H5FP_sap_handle_alloc_request(H5FP_request_t *req)
 
     FUNC_ENTER_NOINIT(H5FP_sap_handle_alloc_request);
 
-    if ((info = H5FP_find_file_info(req->file_id)) != NULL) {
-        sap_alloc.req_id = req->req_id;
-        sap_alloc.file_id = info->file_id;
-        sap_alloc.status = H5FP_STATUS_OK;
-        sap_alloc.mem_type = req->mem_type;
+    sap_alloc.req_id = req->req_id;
+    sap_alloc.file_id = info->file_id;
+    sap_alloc.status = H5FP_STATUS_OK;
+    sap_alloc.mem_type = req->mem_type;
 
+    if ((info = H5FP_find_file_info(req->file_id)) != NULL) {
         /*
          * Try allocating from the free-list that is kept on the server
          * first. If that fails, then call the specified allocation
@@ -1497,19 +1505,94 @@ H5FP_sap_handle_alloc_request(H5FP_request_t *req)
          * Whatta pain.
          */
         if ((sap_alloc.addr = H5FD_alloc((H5FD_t*)&info->file, req->mem_type, H5P_DEFAULT,
-                                         req->meta_block_size)) == HADDR_UNDEF)
+                                         req->meta_block_size)) == HADDR_UNDEF) {
+            sap_alloc.status = H5FP_STATUS_CANT_ALLOC;
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL,
                         "SAP unable to allocate file memory");
+        }
 
         /* Get the EOA from the file. This call doesn't fail. */
         sap_alloc.eoa = ((H5FD_t*)&info->file)->cls->get_eoa((H5FD_t*)&info->file);
+        sap_alloc.status = H5FP_STATUS_OK;
+    } else {
+        sap_alloc.addr = HADDR_UNDEF;
+        sap_alloc.eoa = HADDR_UNDEF;
+        sap_alloc.status = H5FP_STATUS_CANT_ALLOC;
+    }
 
-        if ((mrc = MPI_Send(&sap_alloc, 1, H5FP_alloc, (int)req->proc_rank,
-                            H5FP_TAG_ALLOC, H5FP_SAP_COMM)) != MPI_SUCCESS)
-            HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
+done:
+    if ((mrc = MPI_Send(&sap_alloc, 1, H5FP_alloc, (int)req->proc_rank,
+                        H5FP_TAG_ALLOC, H5FP_SAP_COMM)) != MPI_SUCCESS)
+        HMPI_DONE_ERROR(FAIL, "MPI_Send failed", mrc);
+
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
+ * Function:    H5FP_sap_handle_free_request
+ * Purpose:     Handle a request to free data from the file.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 31. March 2003
+ * Modifications:
+ */
+static herr_t
+H5FP_sap_handle_free_request(H5FP_request_t *req)
+{
+    H5FP_alloc_t    sap_alloc;
+    H5FP_file_info *info;
+    H5FP_status_t   exit_state = H5FP_STATUS_OK;
+    herr_t          ret_value = SUCCEED;
+    int             mrc;
+
+    FUNC_ENTER_NOINIT(H5FP_sap_handle_free_request);
+
+    sap_alloc.req_id = req->req_id;
+    sap_alloc.file_id = info->file_id;
+    sap_alloc.status = H5FP_STATUS_OK;
+    sap_alloc.addr = HADDR_UNDEF;
+
+    if ((info = H5FP_find_file_info(req->file_id)) != NULL) {
+        H5FD_t *f = (H5FD_t*)&info->file;
+
+        if (f->eoma)
+            if (H5FD_free(f, H5FD_MEM_DEFAULT, H5P_DEFAULT,
+                          f->eoma, f->cur_meta_block_size) != SUCCEED) {
+                exit_state = H5FP_STATUS_CANT_FREE;
+                sap_alloc.status = H5FP_STATUS_CANT_FREE;
+                HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
+                            "SAP unable to free metadata block");
+            }
+
+        /* Reset metadata block information, just in case */
+        f->eoma = 0;
+        f->cur_meta_block_size = 0;
+
+        if (f->eosda)
+            if (H5FD_free(f, H5FD_MEM_DRAW, H5P_DEFAULT,
+                          f->eosda, f->cur_sdata_block_size) != SUCCEED) {
+                exit_state = H5FP_STATUS_CANT_FREE;
+                sap_alloc.status = H5FP_STATUS_CANT_FREE;
+                HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
+                            "SAP unable to free 'small data' block");
+            }
+
+        /* Reset "small data" block information, just in case */
+        f->eosda = 0;
+        f->cur_sdata_block_size = 0;
+
+        /* Get the EOA from the file. This call doesn't fail. */
+        sap_alloc.eoa = ((H5FD_t*)&info->file)->cls->get_eoa((H5FD_t*)&info->file);
+    } else {
+        sap_alloc.eoa = HADDR_UNDEF;
+        sap_alloc.status = H5FP_STATUS_CANT_FREE;
     }
 
 done:
+    if ((mrc = MPI_Send(&sap_alloc, 1, H5FP_alloc, (int)req->proc_rank,
+                        H5FP_TAG_ALLOC, H5FP_SAP_COMM)) != MPI_SUCCESS)
+        HMPI_DONE_ERROR(FAIL, "MPI_Send failed", mrc);
+
     FUNC_LEAVE_NOAPI(ret_value);
 }