1.12 New References for Dimension Scales (#1139) (#1574)

* New references for Dimension Scale APIs (#1139)

* Enable usage of new-style references with dimension scale APIs

* Add API to check if an object ID represents a native connector object

* Modified code to use new function H5DSwith_new_ref to determine if new references should be used with
Dimension Scales. The new function return TRUE if non-native connector is used or if H5_DIMENSION_SCALES_WITH_NEW_REF
varible is define at configure time (--enable-dimension-scales-with-new-ref).

Tested on jelly.

ToDo: generate testing file on BE system and enable the test; add flag to CMake; test netCDF-4 with the new references.

* Adding new test files generated on BE system (hedgehog) created by 32 and 64-bit library.

test_ds chokes on test_ds_le_new_ref.h5  on BE system; test passes for test_ds_be_new_ref-32bit.h5
for the 32-bit library and fails for the 64-bit library, and vice versa. I am checking the files for further
investigation; but current implementation of the new references is not portable between LE and BE systems,
and 32 and 64-bit systems.

* Minor fixes for testing issues

* Update test_ds.c

Enabled broken test; tests pass now.

* Update RELEASE.txt

Documented new option to use new references with the HDF5 dimension scales APIs (H5DS*).

* Update MANIFEST for new 32-bit new-style references test file for H5DS APIs

* Update 'dimension scales w/ new-style refs' feature based on review

Co-authored-by: Elena <epourmal@hdfgroup.org>

* Fix H5DS warnings related to new H5DSwith_new_ref and H5VLobject_is_native APIs (#1184)

Co-authored-by: jhendersonHDF <jhenderson@hdfgroup.org>
Co-authored-by: Elena <epourmal@hdfgroup.org>

11 files changed, 684 insertions, 368 deletions

diff --git a/hl/CMakeLists.txt b/hl/CMakeLists.txt
index 5061c6c..9391231 100644
--- a/hl/CMakeLists.txt
+++ b/hl/CMakeLists.txt
@@ -2,6 +2,14 @@ cmake_minimum_required (VERSION 3.12)
 project (HDF5_HL C)
 
 #-----------------------------------------------------------------------------
+# Option to use new-style references with dimension scale APIs
+#-----------------------------------------------------------------------------
+option (HDF5_DIMENSION_SCALES_NEW_REF  "Use new-style references with dimension scale APIs" OFF)
+if (HDF5_DIMENSION_SCALES_NEW_REF)
+  set (H5_DIMENSION_SCALES_WITH_NEW_REF 1)
+endif ()
+
+#-----------------------------------------------------------------------------
 # List Source files
 #-----------------------------------------------------------------------------
 
diff --git a/hl/src/H5DS.c b/hl/src/H5DS.c
index 2bd4046..c947d16 100644
--- a/hl/src/H5DS.c
+++ b/hl/src/H5DS.c
@@ -18,7 +18,40 @@
 
 /* Local routines */
 static herr_t H5DS_is_reserved(hid_t did);
-static hid_t  H5DS_get_REFLIST_type(void);
+
+/*-------------------------------------------------------------------------
+ * Function: H5DSwith_new_ref
+ *
+ * Purpose: Determines if new references are used with dimension scales.
+ *   The function H5DSwith_new_ref takes any object identifier and checks
+ *   if new references are used for dimension scales. Currently,
+ *   new references are used when non-native VOL connector is used or when
+ *   H5_DIMENSION_SCALES_WITH_NEW_REF is set up via configure option.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5DSwith_new_ref(hid_t obj_id, hbool_t *with_new_ref)
+{
+    hbool_t config_flag = FALSE;
+    hbool_t native      = FALSE;
+
+    if (!with_new_ref)
+        return FAIL;
+
+    if (H5VLobject_is_native(obj_id, &native) < 0)
+        return FAIL;
+
+#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
+    config_flag = TRUE;
+#endif
+
+    *with_new_ref = (config_flag || !native);
+
+    return SUCCEED;
+}
 
 /*-------------------------------------------------------------------------
  * Function: H5DSset_scale
@@ -106,22 +139,34 @@ H5DSset_scale(hid_t dsid, const char *dimname)
 herr_t
 H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
 {
-    int         has_dimlist;
-    int         has_reflist;
-    int         is_ds;
-    hssize_t    nelmts;
-    hid_t       sid;          /* space ID */
-    hid_t       tid  = -1;    /* attribute type ID */
-    hid_t       ntid = -1;    /* attribute native type ID */
-    hid_t       aid  = -1;    /* attribute ID */
-    int         rank;         /* rank of dataset */
-    hsize_t     dims[1];      /* dimension of the "REFERENCE_LIST" array */
-    ds_list_t   dsl;          /* attribute data in the DS pointing to the dataset */
-    ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */
-    hobj_ref_t  ref_to_ds;    /* reference to the DS */
-    hobj_ref_t  ref_j;        /* iterator reference */
-    hvl_t *     buf = NULL;   /* VL buffer to store in the attribute */
-    hid_t       dsid_j;       /* DS dataset ID in DIMENSION_LIST */
+    int      has_dimlist;
+    int      has_reflist;
+    int      is_ds;
+    hssize_t nelmts;
+    hid_t    sid, sid_w;             /* space ID */
+    hid_t    tid  = H5I_INVALID_HID; /* attribute type ID */
+    hid_t    ntid = H5I_INVALID_HID; /* attribute native type ID */
+    hid_t    aid  = H5I_INVALID_HID; /* attribute ID */
+    int      rank;                   /* rank of dataset */
+    hsize_t  dims[1];                /* dimension of the "REFERENCE_LIST" array */
+
+    ds_list_t  dsl;          /* attribute data in the DS pointing to the dataset */
+    ds_list_t *dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */
+    ds_list_t *dsbuf_w =
+        NULL; /* array of "REFERENCE_LIST" attribute data to write when adding new reference to a dataset */
+    hobj_ref_t ref_to_ds; /* reference to the DS */
+    hobj_ref_t ref_j;     /* iterator reference */
+
+    /* Variables to be used when new references are used */
+    nds_list_t  ndsl;
+    nds_list_t *ndsbuf   = NULL;
+    nds_list_t *ndsbuf_w = NULL;
+    H5R_ref_t   nref_to_ds;
+    H5R_ref_t   nref_j;
+    hbool_t     is_new_ref;
+
+    hvl_t *     buf = NULL; /* VL buffer to store in the attribute */
+    hid_t       dsid_j;     /* DS dataset ID in DIMENSION_LIST */
     H5O_info2_t oi1, oi2;
     H5I_type_t  it1, it2;
     int         i;
@@ -159,6 +204,14 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             return FAIL;
     } /* end if */
 
+    /*-------------------------------------------------------------------------
+     * determine if old or new references should be used
+     *-------------------------------------------------------------------------
+     */
+
+    if (H5DSwith_new_ref(did, &is_new_ref) < 0)
+        return FAIL;
+
     /* get ID type */
     if ((it1 = H5Iget_type(did)) < 0)
         return FAIL;
@@ -209,14 +262,23 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
      *  and one to the dataset, saved in "REFERENCE_LIST"
      *-------------------------------------------------------------------------
      */
-    /* create a reference for the >>DS<< dataset */
-    if (H5Rcreate(&ref_to_ds, dsid, ".", H5R_OBJECT, (hid_t)-1) < 0)
-        return FAIL;
-
-    /* create a reference for the >>data<< dataset */
-    if (H5Rcreate(&dsl.ref, did, ".", H5R_OBJECT, (hid_t)-1) < 0)
-        return FAIL;
+    if (is_new_ref) {
+        /* create a reference for the >>DS<< dataset */
+        if (H5Rcreate_object(dsid, ".", H5P_DEFAULT, &nref_to_ds) < 0)
+            return FAIL;
+        /* create a reference for the >>data<< dataset */
+        if (H5Rcreate_object(did, ".", H5P_DEFAULT, &ndsl.ref) < 0)
+            return FAIL;
+    }
+    else {
+        /* create a reference for the >>DS<< dataset */
+        if (H5Rcreate(&ref_to_ds, dsid, ".", H5R_OBJECT, (hid_t)-1) < 0)
+            return FAIL;
 
+        /* create a reference for the >>data<< dataset */
+        if (H5Rcreate(&dsl.ref, did, ".", H5R_OBJECT, (hid_t)-1) < 0)
+            return FAIL;
+    }
     /* try to find the attribute "DIMENSION_LIST" on the >>data<< dataset */
     if ((has_dimlist = H5LT_find_attribute(did, DIMENSION_LIST)) < 0)
         return FAIL;
@@ -234,9 +296,14 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             return FAIL;
 
         /* create the type for the attribute "DIMENSION_LIST" */
-        if ((tid = H5Tvlen_create(H5T_STD_REF_OBJ)) < 0)
-            goto out;
-
+        if (is_new_ref) {
+            if ((tid = H5Tvlen_create(H5T_STD_REF)) < 0)
+                goto out;
+        }
+        else {
+            if ((tid = H5Tvlen_create(H5T_STD_REF_OBJ)) < 0)
+                goto out;
+        }
         /* create the attribute */
         if ((aid = H5Acreate2(did, DIMENSION_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
             goto out;
@@ -252,24 +319,32 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
         }
 
         /* store the REF information in the index of the dataset that has the DS */
-        buf[idx].len                  = 1;
-        buf[idx].p                    = HDmalloc(1 * sizeof(hobj_ref_t));
-        ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds;
-
+        buf[idx].len = 1;
+        if (is_new_ref) {
+            buf[idx].p                   = HDmalloc(1 * sizeof(H5R_ref_t));
+            ((H5R_ref_t *)buf[idx].p)[0] = nref_to_ds;
+        }
+        else {
+            buf[idx].p                    = HDmalloc(1 * sizeof(hobj_ref_t));
+            ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds;
+        }
         /* write the attribute with the reference */
         if (H5Awrite(aid, tid, buf) < 0)
             goto out;
 
         /* close */
-        if (H5Treclaim(tid, sid, H5P_DEFAULT, buf) < 0)
-            goto out;
+        if (is_new_ref) {
+            if (H5Rdestroy(&nref_to_ds) < 0)
+                goto out;
+        }
         if (H5Sclose(sid) < 0)
             goto out;
         if (H5Tclose(tid) < 0)
             goto out;
         if (H5Aclose(aid) < 0)
             goto out;
-
+        HDfree(buf[idx].p);
+        buf[idx].p = NULL;
         HDfree(buf);
         buf = NULL;
     }
@@ -302,12 +377,20 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
         /* iterate all the REFs in this dimension IDX */
         for (i = 0; i < (int)buf[idx].len; i++) {
             /* get the reference */
-            ref_j = ((hobj_ref_t *)buf[idx].p)[i];
+            if (is_new_ref) {
+                nref_j = ((H5R_ref_t *)buf[idx].p)[i];
 
-            /* get the scale id for this REF */
-            if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref_j)) < 0)
-                goto out;
+                /* get the scale id for this REF */
+                if ((dsid_j = H5Ropen_object(&nref_j, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                    goto out;
+            }
+            else {
+                ref_j = ((hobj_ref_t *)buf[idx].p)[i];
 
+                /* get the scale id for this REF */
+                if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref_j)) < 0)
+                    goto out;
+            }
             /* get info for DS in the parameter list */
             if (H5Oget_info3(dsid, &oi1, H5O_INFO_BASIC) < 0)
                 goto out;
@@ -335,17 +418,33 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             /* we are adding one more DS to this dimension */
             if (buf[idx].len > 0) {
                 buf[idx].len++;
-                len                                 = buf[idx].len;
-                buf[idx].p                          = HDrealloc(buf[idx].p, len * sizeof(hobj_ref_t));
-                ((hobj_ref_t *)buf[idx].p)[len - 1] = ref_to_ds;
+                len = buf[idx].len;
+                if (is_new_ref) {
+                    buf[idx].p                         = HDrealloc(buf[idx].p, len * sizeof(H5R_ref_t));
+                    ((H5R_ref_t *)buf[idx].p)[len - 1] = nref_to_ds;
+                }
+                else {
+                    buf[idx].p                          = HDrealloc(buf[idx].p, len * sizeof(hobj_ref_t));
+                    ((hobj_ref_t *)buf[idx].p)[len - 1] = ref_to_ds;
+                }
             } /* end if */
             else {
                 /* store the REF information in the index of the dataset that has the DS */
-                buf[idx].len                  = 1;
-                buf[idx].p                    = HDmalloc(sizeof(hobj_ref_t));
-                ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds;
+                buf[idx].len = 1;
+                if (is_new_ref) {
+                    buf[idx].p                   = HDmalloc(sizeof(H5R_ref_t));
+                    ((H5R_ref_t *)buf[idx].p)[0] = nref_to_ds;
+                }
+                else {
+                    buf[idx].p                    = HDmalloc(sizeof(hobj_ref_t));
+                    ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds;
+                }
             } /* end else */
         }     /* end if */
+        else {
+            if (is_new_ref && H5Rdestroy(&nref_to_ds) < 0)
+                goto out;
+        }
 
         /* write the attribute with the new references */
         if (H5Awrite(aid, tid, buf) < 0)
@@ -385,29 +484,40 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             goto out;
 
         /* create the compound datatype for the attribute "REFERENCE_LIST" */
-        if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0)
-            goto out;
-
-        /* insert reference field */
-        if (H5Tinsert(tid, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0)
-            goto out;
-
-        /* insert dimension idx of the dataset field */
-        if (H5Tinsert(tid, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_INT) < 0)
-            goto out;
+        if (is_new_ref) {
+            if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(nds_list_t))) < 0)
+                goto out;
+            if (H5Tinsert(tid, "dataset", HOFFSET(nds_list_t, ref), H5T_STD_REF) < 0)
+                goto out;
+            if (H5Tinsert(tid, "dimension", HOFFSET(nds_list_t, dim_idx), H5T_NATIVE_UINT) < 0)
+                goto out;
+        }
+        else {
+            if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0)
+                goto out;
+            if (H5Tinsert(tid, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0)
+                goto out;
+            if (H5Tinsert(tid, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_UINT) < 0)
+                goto out;
+        }
 
         /* create the attribute */
         if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
             goto out;
 
         /* store the IDX information */
-        dsl.dim_idx = idx;
-
-        /* write the attribute with the reference */
-        if (H5Awrite(aid, tid, &dsl) < 0)
-            goto out;
-
-        /* close */
+        if (is_new_ref) {
+            ndsl.dim_idx = idx;
+            if (H5Awrite(aid, tid, &ndsl) < 0)
+                goto out;
+            if (H5Rdestroy(&ndsl.ref) < 0)
+                goto out;
+        }
+        else {
+            dsl.dim_idx = idx;
+            if (H5Awrite(aid, tid, &dsl) < 0)
+                goto out;
+        }
         if (H5Sclose(sid) < 0)
             goto out;
         if (H5Tclose(tid) < 0)
@@ -421,6 +531,9 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
      *-------------------------------------------------------------------------
      */
     else if (has_reflist == 1) {
+        hid_t tmp_id; /* Temporary DS dataset ID to recreate reference */
+        int   j;
+
         if ((aid = H5Aopen(dsid, REFERENCE_LIST, H5P_DEFAULT)) < 0)
             goto out;
 
@@ -428,7 +541,7 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             goto out;
 
         /* get native type to read attribute REFERENCE_LIST */
-        if ((ntid = H5DS_get_REFLIST_type()) < 0)
+        if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0)
             goto out;
 
         /* get and save the old reference(s) */
@@ -439,17 +552,22 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
             goto out;
 
         nelmts++;
-
-        dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
-        if (dsbuf == NULL)
-            goto out;
-
-        if (H5Aread(aid, ntid, dsbuf) < 0)
-            goto out;
+        if (is_new_ref) {
+            ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t));
+            if (ndsbuf == NULL)
+                goto out;
+            if (H5Aread(aid, ntid, ndsbuf) < 0)
+                goto out;
+        }
+        else {
+            dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
+            if (dsbuf == NULL)
+                goto out;
+            if (H5Aread(aid, ntid, dsbuf) < 0)
+                goto out;
+        }
 
         /* close */
-        if (H5Sclose(sid) < 0)
-            goto out;
         if (H5Aclose(aid) < 0)
             goto out;
 
@@ -458,40 +576,93 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx)
          *-------------------------------------------------------------------------
          */
 
+        /* Allocate new buffer to copy old references and add new one */
+
+        if (is_new_ref) {
+            ndsbuf_w = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t));
+            if (ndsbuf_w == NULL)
+                goto out;
+        }
+        else {
+            dsbuf_w = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
+            if (dsbuf_w == NULL)
+                goto out;
+        }
+        /* Recreate the references we read from the existing "REFERENCE_LIST" attribute */
+        for (j = 0; j < nelmts - 1; j++) {
+            if (is_new_ref) {
+                ndsbuf_w[j].dim_idx = ndsbuf[j].dim_idx;
+                tmp_id              = H5Ropen_object(&ndsbuf[j].ref, H5P_DEFAULT, H5P_DEFAULT);
+                if (tmp_id < 0)
+                    goto out;
+                if (H5Rcreate_object(tmp_id, ".", H5P_DEFAULT, &ndsbuf_w[j].ref) < 0) {
+                    H5Dclose(tmp_id);
+                    goto out;
+                }
+            }
+            else {
+                dsbuf_w[j] = dsbuf[j];
+            }
+        }
+        /* store the IDX information (index of the dataset that has the DS) */
+        if (is_new_ref) {
+            ndsl.dim_idx         = idx;
+            ndsbuf_w[nelmts - 1] = ndsl;
+        }
+        else {
+            dsl.dim_idx         = idx;
+            dsbuf_w[nelmts - 1] = dsl;
+        }
+
         /* the attribute must be deleted, in order to the new one can reflect the changes*/
         if (H5Adelete(dsid, REFERENCE_LIST) < 0)
             goto out;
 
-        /* store the IDX information (index of the dataset that has the DS) */
-        dsl.dim_idx       = idx;
-        dsbuf[nelmts - 1] = dsl;
-
         /* create a new data space for the new references array */
         dims[0] = (hsize_t)nelmts;
 
-        if ((sid = H5Screate_simple(1, dims, NULL)) < 0)
+        if ((sid_w = H5Screate_simple(1, dims, NULL)) < 0)
             goto out;
 
         /* create the attribute again with the changes of space */
-        if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid_w, H5P_DEFAULT, H5P_DEFAULT)) < 0)
             goto out;
 
         /* write the attribute with the new references */
-        if (H5Awrite(aid, ntid, dsbuf) < 0)
-            goto out;
-
-        /* close */
+        if (is_new_ref) {
+            if (H5Awrite(aid, ntid, ndsbuf_w) < 0)
+                goto out;
+            if (H5Treclaim(tid, sid, H5P_DEFAULT, ndsbuf_w) < 0)
+                goto out;
+        }
+        else {
+            if (H5Awrite(aid, ntid, dsbuf_w) < 0)
+                goto out;
+            if (H5Treclaim(tid, sid, H5P_DEFAULT, dsbuf_w) < 0)
+                goto out;
+        }
         if (H5Sclose(sid) < 0)
             goto out;
+        if (H5Sclose(sid_w) < 0)
+            goto out;
         if (H5Tclose(tid) < 0)
             goto out;
         if (H5Aclose(aid) < 0)
             goto out;
         if (H5Tclose(ntid) < 0)
             goto out;
-
-        HDfree(dsbuf);
-        dsbuf = NULL;
+        if (is_new_ref) {
+            HDfree(ndsbuf);
+            dsbuf = NULL;
+            HDfree(ndsbuf_w);
+            dsbuf = NULL;
+        }
+        else {
+            HDfree(dsbuf);
+            dsbuf = NULL;
+            HDfree(dsbuf_w);
+            dsbuf = NULL;
+        }
     } /* has_reflist */
 
     /*-------------------------------------------------------------------------
@@ -515,6 +686,8 @@ out:
         HDfree(buf);
     if (dsbuf)
         HDfree(dsbuf);
+    if (dsbuf_w)
+        HDfree(dsbuf_w);
 
     H5E_BEGIN_TRY
     {
@@ -554,27 +727,35 @@ out:
 herr_t
 H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
 {
-    int         has_dimlist;
-    int         has_reflist;
-    hssize_t    nelmts;
-    hid_t       dsid_j;       /* DS dataset ID in DIMENSION_LIST */
-    hid_t       did_i;        /* dataset ID in REFERENCE_LIST */
-    hid_t       sid;          /* space ID */
-    hid_t       tid  = -1;    /* attribute type ID */
-    hid_t       ntid = -1;    /* attribute native type ID */
-    hid_t       aid  = -1;    /* attribute ID */
-    int         rank;         /* rank of dataset */
-    ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */
-    hsize_t     dims[1];      /* dimension of the "REFERENCE_LIST" array */
-    hobj_ref_t  ref;          /* reference to the DS */
-    hvl_t *     buf = NULL;   /* VL buffer to store in the attribute */
-    int         i;
-    size_t      j;
-    hssize_t    ii;
-    H5O_info2_t did_oi, dsid_oi, tmp_oi;
-    int         found_dset = 0, found_ds = 0;
-    int         have_ds = 0;
-    htri_t      is_scale;
+    int          has_dimlist;
+    int          has_reflist;
+    hssize_t     nelmts;
+    hid_t        dsid_j;                  /* DS dataset ID in DIMENSION_LIST */
+    hid_t        did_i;                   /* dataset ID in REFERENCE_LIST */
+    hid_t        sid   = H5I_INVALID_HID; /* space ID */
+    hid_t        sid_w = H5I_INVALID_HID; /* space ID */
+    hid_t        tid   = H5I_INVALID_HID; /* attribute type ID */
+    hid_t        ntid  = H5I_INVALID_HID; /* attribute native type ID */
+    hid_t        aid   = H5I_INVALID_HID; /* attribute ID */
+    int          rank;                    /* rank of dataset */
+    nds_list_t * ndsbuf   = NULL;         /* array of attribute data in the DS pointing to the dataset */
+    nds_list_t * ndsbuf_w = NULL; /* array of attribute data in the DS pointing to the dataset to write*/
+    ds_list_t *  dsbuf    = NULL; /* array of attribute data in the DS pointing to the dataset */
+    ds_list_t *  dsbuf_w  = NULL; /* array of attribute data in the DS pointing to the dataset to write*/
+    hsize_t      dims[1];         /* dimension of the "REFERENCE_LIST" array */
+    H5R_ref_t    nref;
+    hobj_ref_t   ref;        /* reference to the DS */
+    hvl_t *      buf = NULL; /* VL buffer to store in the attribute */
+    int          i;
+    size_t       j;
+    hssize_t     ii;
+    H5O_info2_t  did_oi, dsid_oi, tmp_oi;
+    int          found_dset = 0, found_ds = 0;
+    int          have_ds = 0;
+    htri_t       is_scale;
+    hbool_t      is_new_ref;
+    unsigned int tmp_idx;
+    hid_t        tmp_id;
 
     /*-------------------------------------------------------------------------
      * parameter checking
@@ -612,9 +793,17 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
     } /* end if */
 
     /*-------------------------------------------------------------------------
-     * Find "DIMENSION_LIST"
+     * determine if old or new references should be used
+     *-------------------------------------------------------------------------
+     */
+    if (H5DSwith_new_ref(did, &is_new_ref) < 0)
+        return FAIL;
+
+    /*-------------------------------------------------------------------------
+     * find "DIMENSION_LIST"
      *-------------------------------------------------------------------------
      */
+
     /* try to find the attribute "DIMENSION_LIST" on the >>data<< dataset */
     if ((has_dimlist = H5LT_find_attribute(did, DIMENSION_LIST)) < 0)
         return FAIL;
@@ -676,13 +865,22 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
     /* reset */
     if (buf[idx].len > 0) {
         for (j = 0; j < buf[idx].len; j++) {
-            /* get the reference */
-            ref = ((hobj_ref_t *)buf[idx].p)[j];
+            if (is_new_ref) {
+                /* get the reference */
+                nref = ((H5R_ref_t *)buf[idx].p)[j];
 
-            /* get the DS id */
-            if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
-                goto out;
+                /* get the scale id for this REF */
+                if ((dsid_j = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                    goto out;
+            }
+            else {
+                /* get the reference */
+                ref = ((hobj_ref_t *)buf[idx].p)[j];
 
+                /* get the DS id */
+                if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
+                    goto out;
+            }
             /* get info for this DS */
             if (H5Oget_info3(dsid_j, &tmp_oi, H5O_INFO_BASIC) < 0)
                 goto out;
@@ -708,8 +906,14 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
 
                     size_t len = buf[idx].len;
 
-                    if (j < len - 1)
-                        ((hobj_ref_t *)buf[idx].p)[j] = ((hobj_ref_t *)buf[idx].p)[len - 1];
+                    if (j < len - 1) {
+                        if (is_new_ref) {
+                            ((H5R_ref_t *)buf[idx].p)[j] = ((H5R_ref_t *)buf[idx].p)[len - 1];
+                        }
+                        else {
+                            ((hobj_ref_t *)buf[idx].p)[j] = ((hobj_ref_t *)buf[idx].p)[len - 1];
+                        }
+                    }
                     len = --buf[idx].len;
                     if (len == 0) {
                         HDfree(buf[idx].p);
@@ -770,7 +974,7 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
         goto out;
 
     /* get native type to read attribute REFERENCE_LIST */
-    if ((ntid = H5DS_get_REFLIST_type()) < 0)
+    if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0)
         goto out;
 
     /* get and save the old reference(s) */
@@ -780,22 +984,65 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
     if ((nelmts = H5Sget_simple_extent_npoints(sid)) < 0)
         goto out;
 
-    dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
-    if (dsbuf == NULL)
-        goto out;
-
-    if (H5Aread(aid, ntid, dsbuf) < 0)
-        goto out;
-
+    if (is_new_ref) {
+        ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t));
+        if (ndsbuf == NULL)
+            goto out;
+        if (H5Aread(aid, ntid, ndsbuf) < 0)
+            goto out;
+        ndsbuf_w = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t));
+        if (ndsbuf_w == NULL)
+            goto out;
+    }
+    else {
+        dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
+        if (dsbuf == NULL)
+            goto out;
+        if (H5Aread(aid, ntid, dsbuf) < 0)
+            goto out;
+        dsbuf_w = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
+        if (dsbuf_w == NULL)
+            goto out;
+    }
+    /* Recreate the references we read from the existing "REFERENCE_LIST" attribute */
+    for (i = 0; i < nelmts; i++) {
+        if (is_new_ref) {
+            ndsbuf_w[i].dim_idx = ndsbuf[i].dim_idx;
+            tmp_id              = H5Ropen_object(&ndsbuf[i].ref, H5P_DEFAULT, H5P_DEFAULT);
+            if (tmp_id < 0)
+                goto out;
+            if (H5Rcreate_object(tmp_id, ".", H5P_DEFAULT, &ndsbuf_w[i].ref) < 0) {
+                H5Dclose(tmp_id);
+                goto out;
+            }
+            H5Dclose(tmp_id);
+        }
+        else {
+            dsbuf_w[i] = dsbuf[i];
+        }
+    }
     for (ii = 0; ii < nelmts; ii++) {
         /* First check if we have the same dimension index */
-        if (idx == dsbuf[ii].dim_idx) {
+        if (is_new_ref) {
+            tmp_idx = ndsbuf_w[ii].dim_idx;
+        }
+        else {
+            tmp_idx = dsbuf_w[ii].dim_idx;
+        }
+        if (idx == tmp_idx) {
             /* get the reference to the dataset */
-            ref = dsbuf[ii].ref;
-
-            /* get the dataset id */
-            if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
-                goto out;
+            if (is_new_ref) {
+                /* get the dataset id */
+                nref = ndsbuf_w[ii].ref;
+                if ((did_i = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                    goto out;
+            }
+            else {
+                /* get the dataset id */
+                ref = dsbuf_w[ii].ref;
+                if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
+                    goto out;
+            }
 
             /* get info for this dataset */
             if (H5Oget_info3(did_i, &tmp_oi, H5O_INFO_BASIC) < 0)
@@ -813,7 +1060,12 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
                     goto out;
                 if (!token_cmp) {
                     /* copy the last one to replace the one which is found */
-                    dsbuf[ii] = dsbuf[nelmts - 1];
+                    if (is_new_ref) {
+                        ndsbuf_w[ii] = ndsbuf_w[nelmts - 1];
+                    }
+                    else {
+                        dsbuf_w[ii] = dsbuf_w[nelmts - 1];
+                    }
                     nelmts--;
                     found_dset = 1;
                     break;
@@ -822,9 +1074,7 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
         }         /* if we have the same dimension index */
     }             /* ii */
 
-    /* close space and attribute */
-    if (H5Sclose(sid) < 0)
-        goto out;
+    /* close attribute */
     if (H5Aclose(aid) < 0)
         goto out;
 
@@ -851,32 +1101,69 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx)
         /* create a new data space for the new references array */
         dims[0] = (hsize_t)nelmts;
 
-        if ((sid = H5Screate_simple(1, dims, NULL)) < 0)
+        if ((sid_w = H5Screate_simple(1, dims, NULL)) < 0)
             goto out;
 
         /* create the attribute again with the changes of space */
-        if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid_w, H5P_DEFAULT, H5P_DEFAULT)) < 0)
             goto out;
 
         /* write the new attribute with the new references */
-        if (H5Awrite(aid, ntid, dsbuf) < 0)
-            goto out;
+        if (is_new_ref) {
+            if (H5Awrite(aid, ntid, ndsbuf_w) < 0)
+                goto out;
+        }
+        else {
+            if (H5Awrite(aid, ntid, dsbuf_w) < 0)
+                goto out;
+        }
 
-        /* close space and attribute */
-        if (H5Sclose(sid) < 0)
-            goto out;
         if (H5Aclose(aid) < 0)
             goto out;
     } /* nelmts */
 
+    /* Free refrences */
+    if (is_new_ref) {
+        if (H5Treclaim(tid, sid, H5P_DEFAULT, ndsbuf) < 0)
+            goto out;
+        if (H5Sclose(sid) < 0)
+            goto out;
+        if (sid_w > 0) {
+            if (H5Treclaim(tid, sid_w, H5P_DEFAULT, ndsbuf_w) < 0)
+                goto out;
+            if (H5Sclose(sid_w) < 0)
+                goto out;
+        }
+    }
+    else {
+        if (H5Treclaim(tid, sid, H5P_DEFAULT, dsbuf) < 0)
+            goto out;
+        if (H5Sclose(sid) < 0)
+            goto out;
+        if (sid_w > 0) {
+            if (H5Treclaim(tid, sid_w, H5P_DEFAULT, dsbuf_w) < 0)
+                goto out;
+            if (H5Sclose(sid_w) < 0)
+                goto out;
+        }
+    }
     /* close type */
     if (H5Tclose(tid) < 0)
         goto out;
     if (H5Tclose(ntid) < 0)
         goto out;
-
-    HDfree(dsbuf);
-    dsbuf = NULL;
+    if (is_new_ref) {
+        HDfree(ndsbuf);
+        HDfree(ndsbuf_w);
+        ndsbuf   = NULL;
+        ndsbuf_w = NULL;
+    }
+    else {
+        HDfree(dsbuf);
+        HDfree(dsbuf_w);
+        dsbuf   = NULL;
+        dsbuf_w = NULL;
+    }
 
     return SUCCEED;
 
@@ -889,6 +1176,14 @@ out:
         H5Tclose(ntid);
         H5Tclose(tid);
 
+        if (ndsbuf) {
+            HDfree(ndsbuf);
+            ndsbuf = NULL;
+        }
+        if (ndsbuf_w) {
+            HDfree(ndsbuf_w);
+            ndsbuf_w = NULL;
+        }
         if (dsbuf) {
             HDfree(dsbuf);
             dsbuf = NULL;
@@ -936,21 +1231,24 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
     int         has_dimlist;
     int         has_reflist;
     hssize_t    nelmts;
-    hid_t       sid;          /* space ID */
-    hid_t       tid  = -1;    /* attribute type ID */
-    hid_t       ntid = -1;    /* attribute native type ID */
-    hid_t       aid  = -1;    /* attribute ID */
-    int         rank;         /* rank of dataset */
-    ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */
-    hobj_ref_t  ref;          /* reference to the DS */
-    hvl_t *     buf = NULL;   /* VL buffer to store in the attribute */
-    hid_t       dsid_j;       /* DS dataset ID in DIMENSION_LIST */
-    hid_t       did_i;        /* dataset ID in REFERENCE_LIST */
+    hid_t       sid;                    /* space ID */
+    hid_t       tid  = H5I_INVALID_HID; /* attribute type ID */
+    hid_t       ntid = H5I_INVALID_HID; /* attribute native type ID */
+    hid_t       aid  = H5I_INVALID_HID; /* attribute ID */
+    int         rank;                   /* rank of dataset */
+    nds_list_t *ndsbuf = NULL;          /* array of attribute data in the DS pointing to the dataset */
+    ds_list_t * dsbuf  = NULL;          /* array of attribute data in the DS pointing to the dataset */
+    H5R_ref_t   nref;                   /* reference to the DS */
+    hobj_ref_t  ref;                    /* reference to the DS */
+    hvl_t *     buf = NULL;             /* VL buffer to store in the attribute */
+    hid_t       dsid_j;                 /* DS dataset ID in DIMENSION_LIST */
+    hid_t       did_i;                  /* dataset ID in REFERENCE_LIST */
     H5O_info2_t oi1, oi2, oi3, oi4;
     H5I_type_t  it1, it2;
     int         i;
     int         found_dset = 0, found_ds = 0;
     htri_t      is_scale;
+    hbool_t     is_new_ref;
 
     /*-------------------------------------------------------------------------
      * parameter checking
@@ -982,6 +1280,14 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
             return FAIL;
     } /* end if */
 
+    /*-------------------------------------------------------------------------
+     * determine if old or new references should be used
+     *-------------------------------------------------------------------------
+     */
+
+    if (H5DSwith_new_ref(did, &is_new_ref) < 0)
+        return FAIL;
+
     /* get ID type */
     if ((it1 = H5Iget_type(did)) < 0)
         return FAIL;
@@ -1042,12 +1348,22 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
 
         /* iterate all the REFs in this dimension IDX */
         for (i = 0; i < (int)buf[idx].len; i++) {
-            /* get the reference */
-            ref = ((hobj_ref_t *)buf[idx].p)[i];
+            if (is_new_ref) {
+                /* get the reference */
+                nref = ((H5R_ref_t *)buf[idx].p)[i];
 
-            /* get the scale id for this REF */
-            if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
-                goto out;
+                /* get the scale id for this REF */
+                if ((dsid_j = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                    goto out;
+            }
+            else {
+                /* get the reference */
+                ref = ((hobj_ref_t *)buf[idx].p)[i];
+
+                /* get the scale id for this REF */
+                if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
+                    goto out;
+            }
 
             /* get info for DS in the parameter list */
             if (H5Oget_info3(dsid, &oi1, H5O_INFO_BASIC) < 0)
@@ -1107,7 +1423,7 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
             goto out;
 
         /* get native type to read REFERENCE_LIST attribute */
-        if ((ntid = H5DS_get_REFLIST_type()) < 0)
+        if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0)
             goto out;
 
         /* get and save the old reference(s) */
@@ -1117,13 +1433,20 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
         if ((nelmts = H5Sget_simple_extent_npoints(sid)) < 0)
             goto out;
 
-        dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
-
-        if (dsbuf == NULL)
-            goto out;
-
-        if (H5Aread(aid, ntid, dsbuf) < 0)
-            goto out;
+        if (is_new_ref) {
+            ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t));
+            if (ndsbuf == NULL)
+                goto out;
+            if (H5Aread(aid, ntid, ndsbuf) < 0)
+                goto out;
+        }
+        else {
+            dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t));
+            if (dsbuf == NULL)
+                goto out;
+            if (H5Aread(aid, ntid, dsbuf) < 0)
+                goto out;
+        }
 
         /*-------------------------------------------------------------------------
          * iterate
@@ -1131,40 +1454,58 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
          */
 
         for (i = 0; i < nelmts; i++) {
-            /* get the reference */
-            ref = dsbuf[i].ref;
 
-            /* the reference was not deleted  */
-            if (ref) {
+            if (is_new_ref) {
+                nref = ndsbuf[i].ref;
+                /* get the dataset id */
+                if ((did_i = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                    goto out;
+            }
+            else {
+                ref = dsbuf[i].ref;
                 /* get the dataset id */
                 if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
                     goto out;
+            }
 
-                /* get info for dataset in the parameter list */
-                if (H5Oget_info3(did, &oi3, H5O_INFO_BASIC) < 0)
-                    goto out;
+            /* get info for dataset in the parameter list */
+            if (H5Oget_info3(did, &oi3, H5O_INFO_BASIC) < 0)
+                goto out;
 
-                /* get info for this dataset */
-                if (H5Oget_info3(did_i, &oi4, H5O_INFO_BASIC) < 0)
-                    goto out;
+            /* get info for this dataset */
+            if (H5Oget_info3(did_i, &oi4, H5O_INFO_BASIC) < 0)
+                goto out;
 
-                /* same object */
-                if (oi3.fileno == oi4.fileno) {
-                    int token_cmp;
+            /* same object */
+            if (oi3.fileno == oi4.fileno) {
+                int token_cmp;
 
-                    if (H5Otoken_cmp(did, &oi3.token, &oi4.token, &token_cmp) < 0)
-                        goto out;
+                if (H5Otoken_cmp(did, &oi3.token, &oi4.token, &token_cmp) < 0)
+                    goto out;
+                if (is_new_ref) {
+                    if (!token_cmp && (idx == ndsbuf[i].dim_idx))
+                        found_dset = 1;
+                }
+                else {
                     if (!token_cmp && (idx == dsbuf[i].dim_idx))
                         found_dset = 1;
-                } /* end if */
+                }
+            } /* end if */
 
-                /* close the dereferenced dataset */
-                if (H5Dclose(did_i) < 0)
-                    goto out;
-            } /* if */
-        }     /* i */
+            /* close the dereferenced dataset */
+            if (H5Dclose(did_i) < 0)
+                goto out;
+        } /* for */
 
         /* close */
+        if (is_new_ref) {
+            if (H5Treclaim(ntid, sid, H5P_DEFAULT, ndsbuf) < 0)
+                goto out;
+        }
+        else {
+            if (H5Treclaim(ntid, sid, H5P_DEFAULT, dsbuf) < 0)
+                goto out;
+        }
         if (H5Sclose(sid) < 0)
             goto out;
         if (H5Tclose(ntid) < 0)
@@ -1174,8 +1515,14 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx)
         if (H5Aclose(aid) < 0)
             goto out;
 
-        HDfree(dsbuf);
-        dsbuf = NULL;
+        if (ndsbuf) {
+            HDfree(ndsbuf);
+            ndsbuf = NULL;
+        }
+        if (dsbuf) {
+            HDfree(dsbuf);
+            dsbuf = NULL;
+        }
     } /* has_reflist */
 
     if (found_ds && found_dset)
@@ -1198,6 +1545,10 @@ out:
         HDfree(buf);
         buf = NULL;
     }
+    if (ndsbuf) {
+        HDfree(ndsbuf);
+        ndsbuf = NULL;
+    }
     if (dsbuf) {
         HDfree(dsbuf);
         dsbuf = NULL;
@@ -1249,17 +1600,19 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi
 {
     hid_t      scale_id;
     int        rank;
-    hobj_ref_t ref;        /* reference to the DS */
-    hid_t      sid;        /* space ID */
-    hid_t      tid = -1;   /* attribute type ID */
-    hid_t      aid = -1;   /* attribute ID */
-    hvl_t *    buf = NULL; /* VL buffer to store in the attribute */
-    H5I_type_t it;         /* ID type */
+    H5R_ref_t  nref;                  /* reference to the DS */
+    hobj_ref_t ref;                   /* reference to the DS */
+    hid_t      sid;                   /* space ID */
+    hid_t      tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t      aid = H5I_INVALID_HID; /* attribute ID */
+    hvl_t *    buf = NULL;            /* VL buffer to store in the attribute */
+    H5I_type_t it;                    /* ID type */
     herr_t     ret_value = 0;
     int        j_idx;
     int        nscales;
     int        has_dimlist;
     int        i;
+    hbool_t    is_new_ref;
 
     /*-------------------------------------------------------------------------
      * parameter checking
@@ -1272,6 +1625,14 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi
     if (H5I_DATASET != it)
         return FAIL;
 
+    /*-------------------------------------------------------------------------
+     * determine if old or new references should be used
+     *-------------------------------------------------------------------------
+     */
+
+    if (H5DSwith_new_ref(did, &is_new_ref) < 0)
+        return FAIL;
+
     /* get the number of scales assotiated with this DIM */
     if ((nscales = H5DSget_num_scales(did, dim)) < 0)
         return FAIL;
@@ -1330,17 +1691,32 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi
 
             /* iterate */
             for (i = j_idx; i < nscales; i++) {
-                /* get the reference */
-                ref = ((hobj_ref_t *)buf[dim].p)[i];
-
-                /* disable error reporting, the ID might refer to a deleted dataset */
-                H5E_BEGIN_TRY
-                {
-                    /* get the DS id */
-                    if ((scale_id = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
-                        goto out;
+                if (is_new_ref) {
+                    /* get the reference */
+                    nref = ((H5R_ref_t *)buf[dim].p)[i];
+
+                    /* disable error reporting, the ID might refer to a deleted dataset */
+                    H5E_BEGIN_TRY
+                    {
+                        /* get the DS id */
+                        if ((scale_id = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+                            goto out;
+                    }
+                    H5E_END_TRY;
+                }
+                else {
+                    /* get the reference */
+                    ref = ((hobj_ref_t *)buf[dim].p)[i];
+
+                    /* disable error reporting, the ID might refer to a deleted dataset */
+                    H5E_BEGIN_TRY
+                    {
+                        /* get the DS id */
+                        if ((scale_id = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0)
+                            goto out;
+                    }
+                    H5E_END_TRY;
                 }
-                H5E_END_TRY;
 
                 /* set the return IDX OUT value at current scale index */
                 if (ds_idx != NULL) {
@@ -1413,12 +1789,12 @@ herr_t
 H5DSset_label(hid_t did, unsigned int idx, const char *label)
 {
     int          has_labels;
-    hid_t        sid = -1; /* space ID */
-    hid_t        tid = -1; /* attribute type ID */
-    hid_t        aid = -1; /* attribute ID */
-    int          rank;     /* rank of dataset */
-    hsize_t      dims[1];  /* dimensions of dataset */
-    H5I_type_t   it;       /* ID type */
+    hid_t        sid = H5I_INVALID_HID; /* space ID */
+    hid_t        tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t        aid = H5I_INVALID_HID; /* attribute ID */
+    int          rank;                  /* rank of dataset */
+    hsize_t      dims[1];               /* dimensions of dataset */
+    H5I_type_t   it;                    /* ID type */
     unsigned int i;
     union {                     /* union is needed to eliminate compiler warnings about */
         char **      buf;       /* discarding the 'const' qualifier in the free */
@@ -1616,12 +1992,12 @@ ssize_t
 H5DSget_label(hid_t did, unsigned int idx, char *label, size_t size)
 {
     int        has_labels;
-    hid_t      sid = -1;   /* space ID */
-    hid_t      tid = -1;   /* attribute type ID */
-    hid_t      aid = -1;   /* attribute ID */
-    int        rank;       /* rank of dataset */
-    char **    buf = NULL; /* buffer to store in the attribute */
-    H5I_type_t it;         /* ID type */
+    hid_t      sid = H5I_INVALID_HID; /* space ID */
+    hid_t      tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t      aid = H5I_INVALID_HID; /* attribute ID */
+    int        rank;                  /* rank of dataset */
+    char **    buf = NULL;            /* buffer to store in the attribute */
+    H5I_type_t it;                    /* ID type */
     size_t     nbytes = 0;
     size_t     copy_len;
     int        i;
@@ -1764,10 +2140,10 @@ out:
 ssize_t
 H5DSget_scale_name(hid_t did, char *name, size_t size)
 {
-    hid_t      aid;      /* attribute ID  */
-    hid_t      tid = -1; /* attribute type ID */
-    hid_t      sid;      /* space ID  */
-    H5I_type_t it;       /* ID type */
+    hid_t      aid = H5I_INVALID_HID; /* attribute ID  */
+    hid_t      tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t      sid = H5I_INVALID_HID; /* space ID  */
+    H5I_type_t it;                    /* ID type */
     size_t     nbytes;
     size_t     copy_len;
     int        has_name;
@@ -1881,13 +2257,13 @@ out:
 htri_t
 H5DSis_scale(hid_t did)
 {
-    hid_t       tid = -1;    /* attribute type ID */
-    hid_t       aid = -1;    /* attribute ID */
-    herr_t      attr_class;  /* has the "CLASS" attribute */
-    htri_t      is_ds = -1;  /* set to "not a dimension scale" */
-    H5I_type_t  it;          /* type of identifier */
-    char *      buf = NULL;  /* buffer to read name of attribute */
-    size_t      string_size; /* size of storage for the attribute */
+    hid_t       tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t       aid = H5I_INVALID_HID; /* attribute ID */
+    herr_t      attr_class;            /* has the "CLASS" attribute */
+    htri_t      is_ds = -1;            /* set to "not a dimension scale" */
+    H5I_type_t  it;                    /* type of identifier */
+    char *      buf = NULL;            /* buffer to read name of attribute */
+    size_t      string_size;           /* size of storage for the attribute */
     H5T_class_t type_class;
     H5T_str_t   strpad;
 
@@ -1995,12 +2371,12 @@ int
 H5DSget_num_scales(hid_t did, unsigned int idx)
 {
     int        has_dimlist;
-    hid_t      sid;        /* space ID */
-    hid_t      tid = -1;   /* attribute type ID */
-    hid_t      aid = -1;   /* attribute ID */
-    int        rank;       /* rank of dataset */
-    hvl_t *    buf = NULL; /* VL buffer to store in the attribute */
-    H5I_type_t it;         /* ID type */
+    hid_t      sid;                   /* space ID */
+    hid_t      tid = H5I_INVALID_HID; /* attribute type ID */
+    hid_t      aid = H5I_INVALID_HID; /* attribute ID */
+    int        rank;                  /* rank of dataset */
+    hvl_t *    buf = NULL;            /* VL buffer to store in the attribute */
+    H5I_type_t it;                    /* ID type */
     int        nscales;
 
     /*-------------------------------------------------------------------------
@@ -2112,12 +2488,12 @@ out:
 static herr_t
 H5DS_is_reserved(hid_t did)
 {
-    int     has_class;
-    hid_t   tid = -1;
-    hid_t   aid = -1;
-    char *  buf;          /* Name of attribute */
-    hsize_t storage_size; /* Size of storage for attribute */
-    herr_t  ret;
+    int    has_class;
+    hid_t  tid = H5I_INVALID_HID;
+    hid_t  aid = H5I_INVALID_HID;
+    char * buf = NULL;  /* Name of attribute */
+    size_t string_size; /* Size of storage for attribute */
+    herr_t ret;
 
     /* try to find the attribute "CLASS" on the dataset */
     if ((has_class = H5LT_find_attribute(did, "CLASS")) < 0)
@@ -2142,10 +2518,10 @@ H5DS_is_reserved(hid_t did)
         goto out;
 
     /* allocate buffer large enough to hold string */
-    if ((storage_size = H5Aget_storage_size(aid)) == 0)
+    if ((string_size = H5Tget_size(tid)) == 0)
         goto out;
 
-    buf = (char *)HDmalloc((size_t)storage_size * sizeof(char) + 1);
+    buf = (char *)HDmalloc((size_t)string_size * sizeof(char));
     if (buf == NULL)
         goto out;
 
@@ -2174,51 +2550,11 @@ H5DS_is_reserved(hid_t did)
 out:
     H5E_BEGIN_TRY
     {
+        if (buf)
+            HDfree(buf);
         H5Tclose(tid);
         H5Aclose(aid);
     }
     H5E_END_TRY;
     return FAIL;
 }
-
-/*-------------------------------------------------------------------------
- * Function: H5DS_get_REFLIST_type
- *
- * Purpose: This is a helper function to return a native type for
- *          the REFERENCE_LIST attribute.
- *
- * Return: Type identifier on success and negative on failure
- *
- * Programmer: Elena Pourmal
- *
- * Date: May 22, 2010
- *
- *-------------------------------------------------------------------------
- */
-static hid_t
-H5DS_get_REFLIST_type(void)
-{
-    hid_t ntid_t = -1;
-
-    /* Build native type that corresponds to compound datatype
-       used to store ds_list_t structure in the REFERENCE_LIST
-       attribute */
-
-    if ((ntid_t = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0)
-        goto out;
-
-    if (H5Tinsert(ntid_t, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0)
-        goto out;
-
-    if (H5Tinsert(ntid_t, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_INT) < 0)
-        goto out;
-
-    return ntid_t;
-out:
-    H5E_BEGIN_TRY
-    {
-        H5Tclose(ntid_t);
-    }
-    H5E_END_TRY;
-    return FAIL;
-}
diff --git a/hl/src/H5DSprivate.h b/hl/src/H5DSprivate.h
index 0403a4c..ede0209 100644
--- a/hl/src/H5DSprivate.h
+++ b/hl/src/H5DSprivate.h
@@ -20,12 +20,18 @@
 /* public LT prototypes			*/
 #include "H5DSpublic.h"
 
-/* attribute type of a DS dataset */
+/* attribute type of a DS dataset when old references are used*/
 typedef struct ds_list_t {
     hobj_ref_t   ref;     /* object reference  */
     unsigned int dim_idx; /* dimension index of the dataset */
 } ds_list_t;
 
+/* attribute type of a DS dataset when new references are used*/
+typedef struct nds_list_t {
+    H5R_ref_t    ref;
+    unsigned int dim_idx; /* dimension index of the dataset */
+} nds_list_t;
+
 /*-------------------------------------------------------------------------
  * private functions
  *-------------------------------------------------------------------------
diff --git a/hl/src/H5DSpublic.h b/hl/src/H5DSpublic.h
index 7306cbc..979a173 100644
--- a/hl/src/H5DSpublic.h
+++ b/hl/src/H5DSpublic.h
@@ -25,6 +25,8 @@ typedef herr_t (*H5DS_iterate_t)(hid_t dset, unsigned dim, hid_t scale, void *vi
 extern "C" {
 #endif
 
+H5_HLDLL herr_t H5DSwith_new_ref(hid_t obj_id, hbool_t *with_new_ref);
+
 H5_HLDLL herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx);
 
 H5_HLDLL herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx);
diff --git a/hl/src/H5LT.c b/hl/src/H5LT.c
index f58b461..e63ec92 100644
--- a/hl/src/H5LT.c
+++ b/hl/src/H5LT.c
@@ -1830,46 +1830,6 @@ H5LTset_attribute_double(hid_t loc_id, const char *obj_name, const char *attr_na
 }
 
 /*-------------------------------------------------------------------------
- * Function: find_attr
- *
- * Purpose: operator function used by H5LT_find_attribute
- *
- * Programmer: Pedro Vicente
- *
- * Date: June 21, 2001
- *
- * Comments:
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-find_attr(H5_ATTR_UNUSED hid_t loc_id, const char *name, H5_ATTR_UNUSED const H5A_info_t *ainfo,
-          void *op_data)
-{
-    int ret = H5_ITER_CONT;
-
-    /* check the arguments */
-    if (name == NULL)
-        return H5_ITER_CONT;
-
-    /* Shut compiler up */
-    (void)loc_id;
-    (void)ainfo;
-
-    /* Define a positive value for return value if the attribute was found. This will
-     * cause the iterator to immediately return that positive value,
-     * indicating short-circuit success
-     */
-
-    if (HDstrncmp(name, (char *)op_data, MAX(HDstrlen((char *)op_data), HDstrlen(name))) == 0)
-        ret = H5_ITER_STOP;
-
-    return ret;
-}
-
-/*-------------------------------------------------------------------------
  * Function: H5LTfind_attribute
  *
  * Purpose: Inquires if an attribute named attr_name exists attached to
@@ -1900,17 +1860,13 @@ H5LTfind_attribute(hid_t loc_id, const char *attr_name)
  *
  * Date: June 21, 2001
  *
- * Comments:
- *  The function uses H5Aiterate2 with the operator function find_attr
- *
  * Return:
- *  Success: The return value of the first operator that
- *              returns non-zero, or zero if all members were
- *              processed with no operator returning non-zero.
+ *  Success: Positive if the attribute exists attached to the
+ *              object loc_id. Zero if the attribute does not
+ *              exist attached to the object loc_id.
  *
  *  Failure: Negative if something goes wrong within the
- *              library, or the negative value returned by one
- *              of the operators.
+ *              library.
  *
  *-------------------------------------------------------------------------
  */
@@ -1923,7 +1879,8 @@ H5_GCC_DIAG_OFF("cast-qual")
 herr_t
 H5LT_find_attribute(hid_t loc_id, const char *attr_name)
 {
-    return H5Aiterate2(loc_id, H5_INDEX_NAME, H5_ITER_INC, NULL, find_attr, (void *)attr_name);
+    htri_t attr_exists = H5Aexists(loc_id, attr_name);
+    return (attr_exists < 0) ? (herr_t)-1 : (attr_exists) ? (herr_t)1 : (herr_t)0;
 }
 H5_GCC_DIAG_ON("cast-qual")
 
diff --git a/hl/test/CMakeTests.cmake b/hl/test/CMakeTests.cmake
index 44208e3..e532253 100644
--- a/hl/test/CMakeTests.cmake
+++ b/hl/test/CMakeTests.cmake
@@ -31,7 +31,9 @@ set (HL_REFERENCE_TEST_FILES
     dslat.txt
     dslon.txt
     test_ds_be.h5
+    test_ds_be_new_ref.h5
     test_ds_le.h5
+    test_ds_le_new_ref.h5
     test_ld.h5
 )
 
diff --git a/hl/test/gen_test_ds.c b/hl/test/gen_test_ds.c
index a56e6cf..d7a9f44 100644
--- a/hl/test/gen_test_ds.c
+++ b/hl/test/gen_test_ds.c
@@ -63,7 +63,11 @@ static int test_foreign_scaleattached(const char *filename);
 #define SCALE_4_NAME "scalename_4_"
 
 #define FILENAME "test_ds_"
-#define FILEEXT  ".h5"
+#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
+#define FILEEXT "_new_ref.h5"
+#else
+#define FILEEXT ".h5"
+#endif
 
 /*-------------------------------------------------------------------------
  * the main program
diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c
index 964e13f..c66ef49 100644
--- a/hl/test/test_ds.c
+++ b/hl/test/test_ds.c
@@ -137,10 +137,15 @@ static int test_attach_detach(void);
 #define DIM0_LABEL "Latitude"
 #define DIM1_LABEL "Longitude"
 
+#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
+#define FOREIGN_FILE1 "test_ds_le_new_ref.h5"
+#define FOREIGN_FILE2 "test_ds_be_new_ref.h5"
+#else
 #define FOREIGN_FILE1 "test_ds_le.h5"
 #define FOREIGN_FILE2 "test_ds_be.h5"
-#define FILENAME      "test_ds"
-#define FILEEXT       ".h5"
+#endif
+#define FILENAME "test_ds"
+#define FILEEXT  ".h5"
 
 #define FILE1 "test_ds3.h5"
 #define FILE2 "test_ds4.h5"
@@ -161,10 +166,15 @@ static int test_attach_detach(void);
 int
 main(void)
 {
-    int nerrors = 0;
+    hid_t file_id = H5I_INVALID_HID;
+    int   nerrors = 0;
 
     /* create file to be used in following tests */
-    if (create_test_file("1") < 0) {
+    if ((file_id = create_test_file("1")) < 0) {
+        nerrors = 1;
+        goto error;
+    }
+    if (H5Fclose(file_id) < 0) {
         nerrors = 1;
         goto error;
     }
@@ -179,7 +189,11 @@ main(void)
     nerrors += test_long_scalenames("1") < 0 ? 1 : 0;
     nerrors += test_float_scalenames("1") < 0 ? 1 : 0;
     nerrors += test_numberofscales("1") < 0 ? 1 : 0;
-    if (create_test_file("2") < 0) {
+    if ((file_id = create_test_file("2")) < 0) {
+        nerrors = 1;
+        goto error;
+    }
+    if (H5Fclose(file_id) < 0) {
         nerrors = 1;
         goto error;
     }
@@ -254,10 +268,9 @@ open_test_file(const char *fileext)
 herr_t
 create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank    = 3;
-    int     rankds  = 1;
-    hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
-    char    buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
+    int     rank                = 3;
+    int     rankds              = 1;
+    hsize_t dims[3]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -275,7 +288,7 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_char(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_char(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_char_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -309,10 +322,9 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
 herr_t
 create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank    = 3;
-    int     rankds  = 1;
-    hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
-    short   buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
+    int     rank                = 3;
+    int     rankds              = 1;
+    hsize_t dims[3]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -330,7 +342,7 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_short(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_short(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_short_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -364,10 +376,9 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
 herr_t
 create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank       = RANK;
-    int     rankds     = 1;
-    hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
-    int     buf[DIM1_SIZE * DIM2_SIZE];
+    int     rank                = RANK;
+    int     rankds              = 1;
+    hsize_t dims[RANK]          = {DIM1_SIZE, DIM2_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     int     s1_wbuf[DIM1_SIZE]  = {10, 20, 30};
@@ -380,7 +391,7 @@ create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_int(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_int(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_int_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -409,7 +420,6 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     int     rank                = 4;
     int     rankds              = 1;
     hsize_t dims[4]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE, DIM4_SIZE};
-    long *  buf                 = NULL;
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -429,12 +439,8 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     long    s43_wbuf[DIM4_SIZE] = {180, 180};
     long    s44_wbuf[DIM4_SIZE] = {280, 280};
 
-    /* Allocate buffer */
-    if (NULL == (buf = (long *)HDmalloc(sizeof(long) * DIM1_SIZE * DIM2_SIZE * DIM3_SIZE * DIM4_SIZE)))
-        goto error;
-
     /* make a dataset */
-    if (H5LTmake_dataset_long(fid, dsname, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_long(fid, dsname, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_long_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -471,23 +477,18 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     else
         goto error;
 
-    HDfree(buf);
-
     return SUCCEED;
 
 error:
-    HDfree(buf);
-
     return FAIL;
 }
 
 herr_t
 create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank       = RANK;
-    int     rankds     = 1;
-    hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
-    float   buf[DIM1_SIZE * DIM2_SIZE];
+    int     rank                = RANK;
+    int     rankds              = 1;
+    hsize_t dims[RANK]          = {DIM1_SIZE, DIM2_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     float   s1_wbuf[DIM1_SIZE]  = {10, 20, 30};
@@ -500,7 +501,7 @@ create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_float(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_float(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_float_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
diff --git a/hl/test/test_ds_be_new_ref-32bit.h5 b/hl/test/test_ds_be_new_ref-32bit.h5
new file mode 100644
index 0000000..ee327e9
--- /dev/null
+++ b/hl/test/test_ds_be_new_ref-32bit.h5
diff --git a/hl/test/test_ds_be_new_ref.h5 b/hl/test/test_ds_be_new_ref.h5
new file mode 100644
index 0000000..eddfa02
--- /dev/null
+++ b/hl/test/test_ds_be_new_ref.h5
diff --git a/hl/test/test_ds_le_new_ref.h5 b/hl/test/test_ds_le_new_ref.h5
new file mode 100644
index 0000000..8625d77
--- /dev/null
+++ b/hl/test/test_ds_le_new_ref.h5