Add new H5R API that abstracts object, region and attribute reference types

Also support references to external files

Add new H5T_REF type and type conversion routines

Support conversion from H5T_REF_OBJ/DSET_REG to H5T_REF

Add H5Treclaim() API to reclaim memory of vlen/reference types

Deprecate H5Dvlen_reclaim()

Fix H5T_vlen_reclaim() and H5T_reclaim() to use private callback

Add H5T_ref_reclaim()

Move previous H5R APIs to H5Rdeprec.c

Clean up H5Ocopy

Separate H5O_copy_expand_ref() to H5Ocopy_ref()

Add support for copying new reference types

Clean up deprecated routines to go through VOL and same code path

Fix return codes in existing trefer.c test

Rename trefer.c to trefer_deprec.c

trefer.c is for new references

Add performance test for trefer

Add additional obj_copy_ref test

Make use of tokens and blobs to store references

Skip blob encoding for object references

Start adding new reference examples

7 files changed, 206 insertions, 12 deletions

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 6e1f79d..9f42f95 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -23,9 +23,11 @@ set (examples
     h5_select
     h5_attribute
     h5_mount
-    h5_reference
+    h5_ref_extern
+    h5_ref_compat
+    h5_reference_deprec
     h5_drivers
-    h5_ref2reg
+    h5_ref2reg_deprec
     h5_extlink
     h5_elink_unix2win
     h5_shared_mesg
diff --git a/examples/Makefile.am b/examples/Makefile.am
index 554ee44..131842c 100644
--- a/examples/Makefile.am
+++ b/examples/Makefile.am
@@ -34,8 +34,9 @@ INSTALL_TOP_FILES = README
 EXAMPLE_PROG = h5_write h5_read h5_extend_write h5_chunk_read h5_compound \
             h5_crtgrpd h5_subset h5_cmprss h5_rdwt h5_crtgrpar h5_extend \
             h5_crtatt h5_crtgrp h5_crtdat \
-            h5_group h5_select h5_attribute h5_mount h5_reference h5_drivers \
-            h5_ref2reg h5_extlink h5_elink_unix2win h5_shared_mesg h5_vds h5_vds-exc \
+            h5_group h5_select h5_attribute h5_mount h5_reference_deprec h5_drivers \
+            h5_ref_extern h5_ref_compat \
+            h5_ref2reg_deprec h5_extlink h5_elink_unix2win h5_shared_mesg h5_vds h5_vds-exc \
             h5_vds-exclim h5_vds-eiger h5_vds-simpleIO h5_vds-percival \
             h5_vds-percival-unlim h5_vds-percival-unlim-maxmin
 TEST_SCRIPT=testh5cc.sh 
@@ -47,8 +48,9 @@ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c   \
             h5_crtgrpd.c h5_subset.c h5_cmprss.c h5_rdwt.c h5_crtgrpar.c \
             h5_extend.c h5_crtatt.c h5_crtgrp.c h5_crtdat.c \
             h5_compound.c h5_group.c h5_select.c h5_attribute.c h5_mount.c \
-            h5_reference.c h5_drivers.c h5_extlink.c h5_elink_unix2win.c   \
-            h5_ref2reg.c h5_shared_mesg.c ph5example.c h5_vds.c h5_vds-exc.c \
+            h5_reference_deprec.c h5_drivers.c h5_extlink.c h5_elink_unix2win.c   \
+            h5_ref_extern.c h5_ref_compat.c \
+            h5_ref2reg_deprec.c h5_shared_mesg.c ph5example.c h5_vds.c h5_vds-exc.c \
             h5_vds-exclim.c h5_vds-eiger.c h5_vds-simpleIO.c h5_vds-percival.c \
             h5_vds-percival-unlim.c h5_vds-percival-unlim-maxmin.c
 
@@ -111,8 +113,10 @@ h5_read: $(srcdir)/h5_read.c
 h5_select: $(srcdir)/h5_select.c
 h5_attribute: $(srcdir)/h5_attribute.c
 h5_mount: $(srcdir)/h5_mount.c
-h5_reference: $(srcdir)/h5_reference.c
-h5_ref2reg: $(srcdir)/h5_ref2reg.c
+h5_ref_compat: $(srcdir)/h5_ref_compat.c
+h5_ref_extern: $(srcdir)/h5_ref_extern.c
+h5_reference_deprec: $(srcdir)/h5_reference_deprec.c
+h5_ref2reg_deprec: $(srcdir)/h5_ref2reg_deprec.c
 h5_drivers: $(srcdir)/h5_drivers.c
 ph5example: $(srcdir)/ph5example.c
 h5_dtransform: $(srcdir)/h5_dtransform.c
diff --git a/examples/h5_ref2reg.c b/examples/h5_ref2reg_deprec.c
index dc2964c..dc2964c 100644
--- a/examples/h5_ref2reg.c
+++ b/examples/h5_ref2reg_deprec.c
diff --git a/examples/h5_ref_compat.c b/examples/h5_ref_compat.c
new file mode 100644
index 0000000..a1fbf96
--- /dev/null
+++ b/examples/h5_ref_compat.c
@@ -0,0 +1,90 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+ /*
+  * The example below illustrates the use of the new API with a file that was
+  * written using the old-style reference API, showing how one can take
+  * advantage of the automatic type conversion from old reference type to new
+  * reference type.
+  */
+
+#include <stdlib.h>
+
+#include "hdf5.h"
+#include <assert.h>
+
+#define H5FILE_NAME "refer_deprec.h5"
+
+#define NDIMS    1    /* Number of dimensions */
+#define BUF_SIZE 4    /* Size of example buffer */
+#define NREFS    1    /* Number of references */
+
+int
+main(void) {
+    hid_t file1, dset1, space1;
+    hsize_t dset1_dims[NDIMS] = { BUF_SIZE };
+    int dset_buf[BUF_SIZE];
+
+    hid_t dset2, space2;
+    hsize_t dset2_dims[NDIMS] = { NREFS };
+    hobj_ref_t ref_buf[NREFS] = { 0 };
+    H5R_ref_t new_ref_buf[NREFS] = { 0 };
+    H5O_type_t obj_type;
+    int i;
+
+    for (i = 0; i < BUF_SIZE; i++)
+        dset_buf[i] = i;
+
+    /* Create file with one dataset and close it */
+    file1 = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+    space1 = H5Screate_simple(NDIMS, dset1_dims, NULL);
+    dset1 = H5Dcreate2(file1, "dataset1", H5T_NATIVE_INT, space1, H5P_DEFAULT,
+                       H5P_DEFAULT, H5P_DEFAULT);
+    H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_buf);
+    H5Dclose(dset1);
+    H5Sclose(space1);
+
+    /**
+     * Create reference to dataset1 with deprecated API
+     * (reminder: there is no destroy call for those references)
+     */
+    H5Rcreate(&ref_buf[0], file1, "dataset1", H5R_OBJECT, H5I_INVALID_HID);
+
+    /* Store reference in separate dataset using deprecated reference type */
+    space2 = H5Screate_simple(NDIMS, dset2_dims, NULL);
+    dset2 = H5Dcreate2(file1, "references", H5T_STD_REF_OBJ, space2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    H5Dwrite(dset2, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_buf);
+    H5Dclose(dset2);
+    H5Sclose(space2);
+    H5Fclose(file1);
+
+    /* Read reference from file using new reference type */
+    file1 = H5Fopen(H5FILE_NAME, H5F_ACC_RDONLY, H5P_DEFAULT);
+    dset2 = H5Dopen2(file1, "references", H5P_DEFAULT);
+    H5Dread(dset2, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, new_ref_buf);
+    H5Dclose(dset2);
+
+    /* Access reference and read dataset data through new API */
+    assert(H5Rget_type((const H5R_ref_t *)&new_ref_buf[0]) == H5R_OBJECT2);
+    H5Rget_obj_type3((const H5R_ref_t *)&new_ref_buf[0], H5P_DEFAULT, &obj_type);
+    assert(obj_type == H5O_TYPE_DATASET);
+    dset1 = H5Ropen_object((const H5R_ref_t *)&new_ref_buf[0], H5P_DEFAULT, H5P_DEFAULT);
+    H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_buf);
+    H5Dclose(dset1);
+    H5Rdestroy(&new_ref_buf[0]);
+
+    for (i = 0; i < BUF_SIZE; i++)
+        assert(dset_buf[i] == i);
+    return 0;
+}
+
diff --git a/examples/h5_ref_extern.c b/examples/h5_ref_extern.c
new file mode 100644
index 0000000..4327a06
--- /dev/null
+++ b/examples/h5_ref_extern.c
@@ -0,0 +1,94 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+ /*
+  * The example below illustrates the use of the new API with files that are
+  * opened read-only. Created references to the objects in that file are
+  * stored into a separate file, and accessed from that file, without the user
+  * explicitly opening the original file that was referenced. 
+  */
+
+#include <stdlib.h>
+
+#include "hdf5.h"
+#include <assert.h>
+
+#define H5FILE_NAME1 "refer_extern1.h5"
+#define H5FILE_NAME2 "refer_extern2.h5"
+
+#define NDIMS    1    /* Number of dimensions */
+#define BUF_SIZE 4    /* Size of example buffer */
+#define NREFS    1    /* Number of references */
+
+int
+main(void) {
+    hid_t file1, dset1, space1;
+    hsize_t dset1_dims[NDIMS] = { BUF_SIZE };
+    int dset_buf[BUF_SIZE];
+
+    hid_t file2, dset2, space2;
+    hsize_t dset2_dims[NDIMS] = { NREFS };
+    H5R_ref_t ref_buf[NREFS] = { 0 };
+    H5O_type_t obj_type;
+    int i;
+
+    for (i = 0; i < BUF_SIZE; i++)
+        dset_buf[i] = i;
+
+    /* Create file with one dataset and close it */
+    file1 = H5Fcreate(H5FILE_NAME1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    space1 = H5Screate_simple(NDIMS, dset1_dims, NULL);
+    dset1 = H5Dcreate2(file1, "dataset1", H5T_NATIVE_INT, space1, H5P_DEFAULT,
+                       H5P_DEFAULT, H5P_DEFAULT);
+    H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_buf);
+    H5Dclose(dset1);
+    H5Sclose(space1);
+    H5Fclose(file1);
+
+    /* Create reference to dataset1 in "refer_extern1.h5" */
+    file1 = H5Fopen(H5FILE_NAME1, H5F_ACC_RDONLY, H5P_DEFAULT);
+    H5Rcreate_object(file1, "dataset1", &ref_buf[0]);
+    H5Fclose(file1);
+
+    /* Store reference in dataset in separate file "refer_extern2.h5" */
+    file2 = H5Fcreate(H5FILE_NAME2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    space2 = H5Screate_simple(NDIMS, dset2_dims, NULL);
+    dset2 = H5Dcreate2(file2, "references", H5T_STD_REF, space2, H5P_DEFAULT,
+                       H5P_DEFAULT, H5P_DEFAULT);
+    H5Dwrite(dset2, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_buf);
+    H5Dclose(dset2);
+    H5Sclose(space2);
+    H5Fclose(file2);
+    H5Rdestroy(&ref_buf[0]);
+
+    /* Read reference back from "refer_extern2.h5" */
+    file2 = H5Fopen(H5FILE_NAME2, H5F_ACC_RDONLY, H5P_DEFAULT);
+    dset2 = H5Dopen2(file2, "references", H5P_DEFAULT);
+    H5Dread(dset2, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_buf);
+    H5Dclose(dset2);
+    H5Fclose(file2);
+
+    /* Access reference and read dataset data without opening original file */
+    assert(H5Rget_type((const H5R_ref_t *)&ref_buf[0]) == H5R_OBJECT2);
+    H5Rget_obj_type3((const H5R_ref_t *)&ref_buf[0], H5P_DEFAULT, &obj_type);
+    assert(obj_type == H5O_TYPE_DATASET);
+    dset1 = H5Ropen_object((const H5R_ref_t *)&ref_buf[0], H5P_DEFAULT, H5P_DEFAULT);
+    H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_buf);
+    H5Dclose(dset1);
+    H5Rdestroy(&ref_buf[0]);
+
+    for (i = 0; i < BUF_SIZE; i++)
+        assert(dset_buf[i] == i);
+
+    return 0;
+}
+
diff --git a/examples/h5_reference.c b/examples/h5_reference_deprec.c
index 32a5f59..32a5f59 100644
--- a/examples/h5_reference.c
+++ b/examples/h5_reference_deprec.c
diff --git a/examples/run-c-ex.sh.in b/examples/run-c-ex.sh.in
index 209cdd7..a70117f 100644
--- a/examples/run-c-ex.sh.in
+++ b/examples/run-c-ex.sh.in
@@ -112,12 +112,16 @@ then
         rm h5_attribute &&\
         RunTest h5_mount &&\
         rm h5_mount &&\
-        RunTest h5_reference &&\
-        rm h5_reference &&\
+        RunTest h5_reference_deprec &&\
+        rm h5_reference_deprec &&\
+        RunTest h5_ref_extern &&\
+        rm h5_ref_extern &&\
+        RunTest h5_ref_compat &&\
+        rm h5_ref_compat &&\
         RunTest h5_drivers &&\
         rm h5_drivers &&\
-        RunTest h5_ref2reg &&\
-        rm h5_ref2reg &&\
+        RunTest h5_ref2reg_deprec &&\
+        rm h5_ref2reg_deprec &&\
         RunTest h5_extlink &&\
         rm h5_extlink &&\
         RunTest h5_elink_unix2win &&\