Clean up memory allocated when reading messages in H5Dlayout on error (#2811)

2 files changed, 26 insertions, 5 deletions

diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 1e0af89..5856d81 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -123,6 +123,16 @@ New Features
 
     Java Library:
     -------------
+    - Fixed memory leaks that could occur when reading a dataset from a
+      malformed file
+
+      When attempting to read layout, pline, and efl information for a
+      dataset, memory leaks could occur if attempting to read pline/efl
+      information threw an error, which is due to the memory that was 
+      allocated for pline and efl not being properly cleaned up on error.  
+	
+      Fixes Github issue #2602
+
     - HDF5GroupInfo class has been deprecated.
 
       This class assumes that an object can contain four values which uniquely identify an
diff --git a/src/H5Dlayout.c b/src/H5Dlayout.c
index 53943b4..019ead1 100644
--- a/src/H5Dlayout.c
+++ b/src/H5Dlayout.c
@@ -591,7 +591,9 @@ herr_t
 H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist)
 {
     htri_t  msg_exists;              /* Whether a particular type of message exists */
+    hbool_t pline_copied  = FALSE;   /* Flag to indicate that dcpl_cache.pline's message was copied */
     hbool_t layout_copied = FALSE;   /* Flag to indicate that layout message was copied */
+    hbool_t efl_copied    = FALSE;   /* Flag to indicate that the EFL message was copied */
     herr_t  ret_value     = SUCCEED; /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -607,6 +609,7 @@ H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist)
         /* Retrieve the I/O pipeline message */
         if (NULL == H5O_msg_read(&(dataset->oloc), H5O_PLINE_ID, &dataset->shared->dcpl_cache.pline))
             HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve message")
+        pline_copied = TRUE;
 
         /* Set the I/O pipeline info in the property list */
         if (H5P_set(plist, H5O_CRT_PIPELINE_NAME, &dataset->shared->dcpl_cache.pline) < 0)
@@ -627,9 +630,10 @@ H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist)
     if ((msg_exists = H5O_msg_exists(&(dataset->oloc), H5O_EFL_ID)) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if message exists")
     if (msg_exists) {
-        /* Retrieve the EFL  message */
+        /* Retrieve the EFL message */
         if (NULL == H5O_msg_read(&(dataset->oloc), H5O_EFL_ID, &dataset->shared->dcpl_cache.efl))
             HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve message")
+        efl_copied = TRUE;
 
         /* Set the EFL info in the property list */
         if (H5P_set(plist, H5D_CRT_EXT_FILE_LIST_NAME, &dataset->shared->dcpl_cache.efl) < 0)
@@ -661,10 +665,17 @@ H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist)
             HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to set chunk sizes")
 
 done:
-    if (ret_value < 0 && layout_copied)
-        if (H5O_msg_reset(H5O_LAYOUT_ID, &dataset->shared->layout) < 0)
-            HDONE_ERROR(H5E_DATASET, H5E_CANTRESET, FAIL, "unable to reset layout info")
-
+    if (ret_value < 0) {
+        if (pline_copied)
+            if (H5O_msg_reset(H5O_PLINE_ID, &dataset->shared->dcpl_cache.pline) < 0)
+                HDONE_ERROR(H5E_DATASET, H5E_CANTRESET, FAIL, "unable to reset pipeline info")
+        if (layout_copied)
+            if (H5O_msg_reset(H5O_LAYOUT_ID, &dataset->shared->layout) < 0)
+                HDONE_ERROR(H5E_DATASET, H5E_CANTRESET, FAIL, "unable to reset layout info")
+        if (efl_copied)
+            if (H5O_msg_reset(H5O_EFL_ID, &dataset->shared->dcpl_cache.efl) < 0)
+                HDONE_ERROR(H5E_DATASET, H5E_CANTRESET, FAIL, "unable to reset efl message")
+    }
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__layout_oh_read() */