Merge pull request #2715 in HDFFV/hdf5 from ~JSOUMAGNE/hdf5:fix_vlen_ref to develop

* commit 'ef6db167a86e6c065d46963dbd75cd325fe83813':
  H5R: fix encoding of references that are part of compound types

4 files changed, 35 insertions, 4 deletions

diff --git a/src/H5Rint.c b/src/H5Rint.c
index 6dfbf2b..0acc887 100644
--- a/src/H5Rint.c
+++ b/src/H5Rint.c
@@ -86,7 +86,7 @@
 } while(0)
 
 /* Debug */
-//#define H5R_DEBUG
+// #define H5R_DEBUG
 #ifdef H5R_DEBUG
 #define H5R_LOG_DEBUG(...) do {                                 \
       HDfprintf(stdout, " # %s(): ", __func__);                 \
diff --git a/src/H5T.c b/src/H5T.c
index 25c1f75..991095d 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -5718,9 +5718,9 @@ H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
                 break;
 
             case H5T_VLEN: /* Recurse on the VL information if it's VL, compound or array, then free VL sequence */
-                /* Recurse if it's VL, compound, enum or array */
+                /* Recurse if it's VL, compound, enum or array (ignore references here so that we can encode them as part of the same blob)*/
                 /* (If the force_conv flag is _not_ set, the type cannot change in size, so don't recurse) */
-                if(dt->shared->parent->shared->force_conv && H5T_IS_COMPLEX(dt->shared->parent->shared->type)) {
+                if(dt->shared->parent->shared->force_conv && H5T_IS_COMPLEX(dt->shared->parent->shared->type) && (dt->shared->parent->shared->type != H5T_REFERENCE)) {
                     if((changed = H5T_set_loc(dt->shared->parent, file, loc)) < 0)
                         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "Unable to set VL location");
                     if(changed > 0)
diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h
index 600d7aa..3eaec8c 100644
--- a/src/H5Tpkg.h
+++ b/src/H5Tpkg.h
@@ -49,7 +49,7 @@
 #define H5T_NAMELEN		32
 
 /* Macro to ease detecting "complex" datatypes (i.e. those with base types or fields) */
-#define H5T_IS_COMPLEX(t)       ((t) == H5T_COMPOUND || (t) == H5T_ENUM || (t) == H5T_VLEN || (t) == H5T_ARRAY)
+#define H5T_IS_COMPLEX(t)       ((t) == H5T_COMPOUND || (t) == H5T_ENUM || (t) == H5T_VLEN || (t) == H5T_ARRAY || (t) == H5T_REFERENCE)
 
 /* Macro to ease detecting fixed "string" datatypes */
 #define H5T_IS_FIXED_STRING(dt) (H5T_STRING == (dt)->type)
diff --git a/src/H5Tref.c b/src/H5Tref.c
index 7c59e75..42e3da2 100644
--- a/src/H5Tref.c
+++ b/src/H5Tref.c
@@ -40,6 +40,19 @@
 #define H5T_REF_OBJ_DISK_SIZE(f)        (H5F_SIZEOF_ADDR(f))
 #define H5T_REF_DSETREG_DISK_SIZE(f)    (H5HG_HEAP_ID_SIZE(f))
 
+/* Debug */
+// #define H5T_REF_DEBUG
+#ifdef H5T_REF_DEBUG
+#define H5T_REF_LOG_DEBUG(...) do {                                 \
+      HDfprintf(stdout, " # %s(): ", __func__);                 \
+      HDfprintf(stdout, __VA_ARGS__);                           \
+      HDfprintf(stdout, "\n");                                  \
+      HDfflush(stdout);                                         \
+  } while (0)
+#else
+#define H5T_REF_LOG_DEBUG(...) do { } while (0)
+#endif
+
 /******************/
 /* Local Typedefs */
 /******************/
@@ -133,6 +146,7 @@ H5T__ref_set_loc(const H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
     htri_t ret_value = FALSE; /* Indicate success, but no location change */
 
     FUNC_ENTER_PACKAGE
+    H5T_REF_LOG_DEBUG("loc=%d", (int)loc);
 
     HDassert(dt);
     /* f is NULL when loc == H5T_LOC_MEMORY */
@@ -319,6 +333,7 @@ H5T__ref_mem_isnull(const H5VL_object_t H5_ATTR_UNUSED *src_file,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC_NOERR
+    H5T_REF_LOG_DEBUG("");
 
     /* Check parameters */
     HDassert(src_buf);
@@ -346,6 +361,7 @@ H5T__ref_mem_setnull(H5VL_object_t H5_ATTR_UNUSED *dst_file, void *dst_buf,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC_NOERR
+    H5T_REF_LOG_DEBUG("");
 
     HDmemset(dst_buf, 0, H5T_REF_MEM_SIZE);
 
@@ -373,6 +389,7 @@ H5T__ref_mem_getsize(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf
     size_t ret_value = 0;               /* Return value */
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Sanity check */
     HDassert(src_buf);
@@ -471,6 +488,7 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Sanity check */
     HDassert(src_buf);
@@ -557,6 +575,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Sanity check */
     HDassert(src_buf);
@@ -670,6 +689,7 @@ H5T__ref_disk_isnull(const H5VL_object_t *src_file, const void *src_buf,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Check parameters */
     HDassert(src_file);
@@ -712,6 +732,7 @@ H5T__ref_disk_setnull(H5VL_object_t *dst_file, void *dst_buf, void *bg_buf)
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(dst_file);
     HDassert(dst_buf);
@@ -761,6 +782,7 @@ H5T__ref_disk_getsize(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_bu
     size_t ret_value = 0;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_buf);
 
@@ -810,6 +832,7 @@ H5T__ref_disk_read(H5VL_object_t *src_file, const void *src_buf, size_t H5_ATTR_
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_file);
     HDassert(src_buf);
@@ -856,6 +879,7 @@ H5T__ref_disk_write(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_buf);
     HDassert(src_size);
@@ -915,6 +939,7 @@ static herr_t H5T__ref_obj_disk_isnull(const H5VL_object_t *src_file,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Check parameters */
     HDassert(src_file);
@@ -967,6 +992,7 @@ H5T__ref_obj_disk_getsize(H5VL_object_t *src_file, const void H5_ATTR_UNUSED *sr
     size_t ret_value = 0;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_file);
     HDassert(src_buf);
@@ -1014,6 +1040,7 @@ H5T__ref_obj_disk_read(H5VL_object_t *src_file, const void *src_buf, size_t src_
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_file);
     HDassert(src_buf);
@@ -1068,6 +1095,7 @@ H5T__ref_dsetreg_disk_isnull(const H5VL_object_t *src_file, const void *src_buf,
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     /* Check parameters */
     HDassert(src_file);
@@ -1123,6 +1151,7 @@ H5T__ref_dsetreg_disk_getsize(H5VL_object_t H5_ATTR_UNUSED *src_file,
 #else
     FUNC_ENTER_STATIC_NOERR
 #endif
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_buf);
 
@@ -1171,6 +1200,7 @@ H5T__ref_dsetreg_disk_read(H5VL_object_t *src_file, const void *src_buf, size_t
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_STATIC
+    H5T_REF_LOG_DEBUG("");
 
     HDassert(src_file);
     HDassert(src_buf);
@@ -1221,6 +1251,7 @@ H5T_ref_reclaim(void *elem, const H5T_t *dt)
     herr_t ret_value = SUCCEED;     /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
+    H5T_REF_LOG_DEBUG("");
 
     /* Sanity checks */
     HDassert(elem);