New references for Dimension Scale APIs (#1139)

* Enable usage of new-style references with dimension scale APIs

* Add API to check if an object ID represents a native connector object

* Modified code to use new function H5DSwith_new_ref to determine if new references should be used with
Dimension Scales. The new function return TRUE if non-native connector is used or if H5_DIMENSION_SCALES_WITH_NEW_REF
varible is define at configure time (--enable-dimension-scales-with-new-ref).

Tested on jelly.

ToDo: generate testing file on BE system and enable the test; add flag to CMake; test netCDF-4 with the new references.

* Adding new test files generated on BE system (hedgehog) created by 32 and 64-bit library.

test_ds chokes on test_ds_le_new_ref.h5  on BE system; test passes for test_ds_be_new_ref-32bit.h5
for the 32-bit library and fails for the 64-bit library, and vice versa. I am checking the files for further
investigation; but current implementation of the new references is not portable between LE and BE systems,
and 32 and 64-bit systems.

* Minor fixes for testing issues

* Update test_ds.c

Enabled broken test; tests pass now.

* Update RELEASE.txt

Documented new option to use new references with the HDF5 dimension scales APIs (H5DS*).

* Update MANIFEST for new 32-bit new-style references test file for H5DS APIs

* Update 'dimension scales w/ new-style refs' feature based on review

Co-authored-by: Elena <epourmal@hdfgroup.org>

6 files changed, 43 insertions, 36 deletions

diff --git a/hl/test/CMakeTests.cmake b/hl/test/CMakeTests.cmake
index 44208e3..e532253 100644
--- a/hl/test/CMakeTests.cmake
+++ b/hl/test/CMakeTests.cmake
@@ -31,7 +31,9 @@ set (HL_REFERENCE_TEST_FILES
     dslat.txt
     dslon.txt
     test_ds_be.h5
+    test_ds_be_new_ref.h5
     test_ds_le.h5
+    test_ds_le_new_ref.h5
     test_ld.h5
 )
 
diff --git a/hl/test/gen_test_ds.c b/hl/test/gen_test_ds.c
index a56e6cf..d7a9f44 100644
--- a/hl/test/gen_test_ds.c
+++ b/hl/test/gen_test_ds.c
@@ -63,7 +63,11 @@ static int test_foreign_scaleattached(const char *filename);
 #define SCALE_4_NAME "scalename_4_"
 
 #define FILENAME "test_ds_"
-#define FILEEXT  ".h5"
+#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
+#define FILEEXT "_new_ref.h5"
+#else
+#define FILEEXT ".h5"
+#endif
 
 /*-------------------------------------------------------------------------
  * the main program
diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c
index 964e13f..c66ef49 100644
--- a/hl/test/test_ds.c
+++ b/hl/test/test_ds.c
@@ -137,10 +137,15 @@ static int test_attach_detach(void);
 #define DIM0_LABEL "Latitude"
 #define DIM1_LABEL "Longitude"
 
+#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
+#define FOREIGN_FILE1 "test_ds_le_new_ref.h5"
+#define FOREIGN_FILE2 "test_ds_be_new_ref.h5"
+#else
 #define FOREIGN_FILE1 "test_ds_le.h5"
 #define FOREIGN_FILE2 "test_ds_be.h5"
-#define FILENAME      "test_ds"
-#define FILEEXT       ".h5"
+#endif
+#define FILENAME "test_ds"
+#define FILEEXT  ".h5"
 
 #define FILE1 "test_ds3.h5"
 #define FILE2 "test_ds4.h5"
@@ -161,10 +166,15 @@ static int test_attach_detach(void);
 int
 main(void)
 {
-    int nerrors = 0;
+    hid_t file_id = H5I_INVALID_HID;
+    int   nerrors = 0;
 
     /* create file to be used in following tests */
-    if (create_test_file("1") < 0) {
+    if ((file_id = create_test_file("1")) < 0) {
+        nerrors = 1;
+        goto error;
+    }
+    if (H5Fclose(file_id) < 0) {
         nerrors = 1;
         goto error;
     }
@@ -179,7 +189,11 @@ main(void)
     nerrors += test_long_scalenames("1") < 0 ? 1 : 0;
     nerrors += test_float_scalenames("1") < 0 ? 1 : 0;
     nerrors += test_numberofscales("1") < 0 ? 1 : 0;
-    if (create_test_file("2") < 0) {
+    if ((file_id = create_test_file("2")) < 0) {
+        nerrors = 1;
+        goto error;
+    }
+    if (H5Fclose(file_id) < 0) {
         nerrors = 1;
         goto error;
     }
@@ -254,10 +268,9 @@ open_test_file(const char *fileext)
 herr_t
 create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank    = 3;
-    int     rankds  = 1;
-    hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
-    char    buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
+    int     rank                = 3;
+    int     rankds              = 1;
+    hsize_t dims[3]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -275,7 +288,7 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_char(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_char(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_char_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -309,10 +322,9 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
 herr_t
 create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank    = 3;
-    int     rankds  = 1;
-    hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
-    short   buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
+    int     rank                = 3;
+    int     rankds              = 1;
+    hsize_t dims[3]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -330,7 +342,7 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_short(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_short(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_short_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -364,10 +376,9 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
 herr_t
 create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank       = RANK;
-    int     rankds     = 1;
-    hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
-    int     buf[DIM1_SIZE * DIM2_SIZE];
+    int     rank                = RANK;
+    int     rankds              = 1;
+    hsize_t dims[RANK]          = {DIM1_SIZE, DIM2_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     int     s1_wbuf[DIM1_SIZE]  = {10, 20, 30};
@@ -380,7 +391,7 @@ create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_int(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_int(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_int_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -409,7 +420,6 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     int     rank                = 4;
     int     rankds              = 1;
     hsize_t dims[4]             = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE, DIM4_SIZE};
-    long *  buf                 = NULL;
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     hsize_t s3_dim[1]           = {DIM3_SIZE};
@@ -429,12 +439,8 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     long    s43_wbuf[DIM4_SIZE] = {180, 180};
     long    s44_wbuf[DIM4_SIZE] = {280, 280};
 
-    /* Allocate buffer */
-    if (NULL == (buf = (long *)HDmalloc(sizeof(long) * DIM1_SIZE * DIM2_SIZE * DIM3_SIZE * DIM4_SIZE)))
-        goto error;
-
     /* make a dataset */
-    if (H5LTmake_dataset_long(fid, dsname, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_long(fid, dsname, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_long_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@@ -471,23 +477,18 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
     else
         goto error;
 
-    HDfree(buf);
-
     return SUCCEED;
 
 error:
-    HDfree(buf);
-
     return FAIL;
 }
 
 herr_t
 create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
 {
-    int     rank       = RANK;
-    int     rankds     = 1;
-    hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
-    float   buf[DIM1_SIZE * DIM2_SIZE];
+    int     rank                = RANK;
+    int     rankds              = 1;
+    hsize_t dims[RANK]          = {DIM1_SIZE, DIM2_SIZE};
     hsize_t s1_dim[1]           = {DIM1_SIZE};
     hsize_t s2_dim[1]           = {DIM2_SIZE};
     float   s1_wbuf[DIM1_SIZE]  = {10, 20, 30};
@@ -500,7 +501,7 @@ create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
     HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
 
     /* make a dataset */
-    if (H5LTmake_dataset_float(fid, name, rank, dims, buf) >= 0) {
+    if (H5LTmake_dataset_float(fid, name, rank, dims, NULL) >= 0) {
         if (fulldims == 0) {
             /* make a DS dataset for the first dimension */
             if (create_DS1_float_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
diff --git a/hl/test/test_ds_be_new_ref-32bit.h5 b/hl/test/test_ds_be_new_ref-32bit.h5
new file mode 100644
index 0000000..ee327e9
--- /dev/null
+++ b/hl/test/test_ds_be_new_ref-32bit.h5
diff --git a/hl/test/test_ds_be_new_ref.h5 b/hl/test/test_ds_be_new_ref.h5
new file mode 100644
index 0000000..eddfa02
--- /dev/null
+++ b/hl/test/test_ds_be_new_ref.h5
diff --git a/hl/test/test_ds_le_new_ref.h5 b/hl/test/test_ds_le_new_ref.h5
new file mode 100644
index 0000000..8625d77
--- /dev/null
+++ b/hl/test/test_ds_le_new_ref.h5