Merge S3 code from develop to 1.10

5 files changed, 109 insertions, 10 deletions

diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
index 3bd0553..ab1fa89 100644
--- a/config/cmake/ConfigureChecks.cmake
+++ b/config/cmake/ConfigureChecks.cmake
@@ -33,7 +33,7 @@ MARK_AS_ADVANCED (HDF5_STRICT_FORMAT_CHECKS)
 # conversions.  If not, some hard conversions will still be prefered even
 # though the data may be wrong (for example, some compilers don't
 # support denormalized floating values) to maximize speed.
-#
+#-----------------------------------------------------------------------------
 option (HDF5_WANT_DATA_ACCURACY "IF data accuracy is guaranteed during data conversions" ON)
 if (HDF5_WANT_DATA_ACCURACY)
   set (${HDF_PREFIX}_WANT_DATA_ACCURACY 1)
@@ -45,7 +45,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DATA_ACCURACY)
 # checked and data conversion exceptions are returned.  This is mainly
 # for the speed optimization of hard conversions.  Soft conversions can
 # actually benefit little.
-#
+#-----------------------------------------------------------------------------
 option (HDF5_WANT_DCONV_EXCEPTION "exception handling functions is checked during data conversions" ON)
 if (HDF5_WANT_DCONV_EXCEPTION)
   set (${HDF_PREFIX}_WANT_DCONV_EXCEPTION 1)
@@ -54,7 +54,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DCONV_EXCEPTION)
 
 # ----------------------------------------------------------------------
 # Check if they would like the function stack support compiled in
-#
+#-----------------------------------------------------------------------------
 option (HDF5_ENABLE_CODESTACK "Enable the function stack tracing (for developer debugging)." OFF)
 if (HDF5_ENABLE_CODESTACK)
   set (${HDF_PREFIX}_HAVE_CODESTACK 1)
@@ -75,7 +75,7 @@ set (${HDF_PREFIX}_HAVE_TMPFILE 1)
 # TODO --------------------------------------------------------------------------
 # Should the Default Virtual File Driver be compiled?
 # This is hard-coded now but option should added to match configure
-#
+#-----------------------------------------------------------------------------
 set (${HDF_PREFIX}_DEFAULT_VFD H5FD_SEC2)
 
 if (NOT DEFINED "${HDF_PREFIX}_DEFAULT_PLUGINDIR")
@@ -92,6 +92,7 @@ if (WINDOWS)
   # Set the flag to indicate that the machine has window style pathname,
   # that is, "drive-letter:\" (e.g. "C:") or "drive-letter:/" (e.g. "C:/").
   # (This flag should be _unset_ for all machines, except for Windows)
+  #-----------------------------------------------------------------------
   set (${HDF_PREFIX}_HAVE_WINDOW_PATH 1)
 endif ()
 
@@ -155,6 +156,26 @@ if (NOT WINDOWS)
 endif ()
 
 #-----------------------------------------------------------------------------
+#  Check if ROS3 driver can be built
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
+  if (HDF5_ENABLE_ROS3_VFD)
+    # CMake version 3.13 fixed FindCURL module
+    if(CMAKE_VERSION VERSION_LESS "3.13.0" AND WIN32)
+        MESSAGE(FATAL_ERROR "Windows builds for this option requires a minimum of CMake 3.13")
+    endif ()
+    find_package(CURL REQUIRED)
+    find_package(OpenSSL REQUIRED)
+    if (${CURL_FOUND} AND ${OPENSSL_FOUND})
+      set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
+      list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES})
+      INCLUDE_DIRECTORIES (${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR})
+    else ()
+      message (STATUS "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
+    endif ()
+endif ()
+
+#-----------------------------------------------------------------------------
 # Check if C has __float128 extension
 #-----------------------------------------------------------------------------
 
@@ -217,7 +238,7 @@ endmacro ()
 # is 0x004733ce17af227f, not the same as the library's conversion to 0x004733ce17af2282.
 # The machine's conversion gets the correct value.  We define the macro and disable
 # this kind of test until we figure out what algorithm they use.
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL  "Checking IF your system converts long double to (unsigned) long values with special algorithm")
 # ----------------------------------------------------------------------
 # Set the flag to indicate that the machine is using a special algorithm
@@ -226,7 +247,7 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL  "Checking IF your syst
 # when the bit sequences are 003fff..., 007fff..., 00ffff..., 01ffff...,
 # ..., 7fffff..., the compiler uses a unknown algorithm.  We define a
 # macro and skip the test for now until we know about the algorithm.
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your system can convert (unsigned) long to long double values with special algorithm")
 # ----------------------------------------------------------------------
 # Set the flag to indicate that the machine can accurately convert
@@ -236,7 +257,7 @@ H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your syste
 # start to go wrong on these two machines.  Adjusting it higher to
 # 0x4351ccf385ebc8a0dfcc... or 0x4351ccf385ebc8a0ffcc... will make the converted
 # values wildly wrong.  This test detects this wrong behavior and disable the test.
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctly converting long double to (unsigned) long long values")
 # ----------------------------------------------------------------------
 # Set the flag to indicate that the machine can accurately convert
@@ -244,14 +265,14 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctl
 # all machines, except for Mac OS 10.4, when the bit sequences are 003fff...,
 # 007fff..., 00ffff..., 01ffff..., ..., 7fffff..., the converted values are twice
 # as big as they should be.
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_LLONG_TO_LDOUBLE_CORRECT "Checking IF correctly converting (unsigned) long long to long double values")
 # ----------------------------------------------------------------------
 # Set the flag to indicate that the machine can accurately convert
 # some long double values
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_DISABLE_SOME_LDOUBLE_CONV "Checking IF the cpu is power9 and cannot correctly converting long double values")
 # ----------------------------------------------------------------------
 # Check if pointer alignments are enforced
-#
+#-----------------------------------------------------------------------------
 H5ConversionTests (${HDF_PREFIX}_NO_ALIGNMENT_RESTRICTIONS "Checking IF alignment restrictions are strictly enforced")
diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in
index 9ccfae4..6cbfd29 100644
--- a/config/cmake/H5pubconf.h.in
+++ b/config/cmake/H5pubconf.h.in
@@ -110,6 +110,9 @@
 /* Define if the function stack tracing code is to be compiled in */
 #cmakedefine H5_HAVE_CODESTACK @H5_HAVE_CODESTACK@
 
+/* Define to 1 if you have the <curl/curl.h> header file. */
+#cmakedefine H5_HAVE_CURL_H @H5_HAVE_CURL_H@
+
 /* Define if Darwin or Mac OS X */
 #cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@
 
@@ -185,6 +188,9 @@
 /* Define to 1 if you have the `gettimeofday' function. */
 #cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@
 
+/* Define to 1 if you have the <hdfs.h> header file. */
+#cmakedefine H5_HAVE_HDFS_H @H5_HAVE_HDFS_H@
+
 /* Define if the compiler understands inline */
 #cmakedefine H5_HAVE_INLINE @H5_HAVE_INLINE@
 
@@ -201,12 +207,24 @@
 /* Define to 1 if you have the <io.h> header file. */
 #cmakedefine H5_HAVE_IO_H @H5_HAVE_IO_H@
 
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#cmakedefine H5_HAVE_LIBCRYPTO @H5_HAVE_LIBCRYPTO@
+
+/* Define to 1 if you have the `curl' library (-lcurl). */
+#cmakedefine H5_HAVE_LIBCURL @H5_HAVE_LIBCURL@
+
 /* Define to 1 if you have the `dl' library (-ldl). */
 #cmakedefine H5_HAVE_LIBDL @H5_HAVE_LIBDL@
 
 /* Define to 1 if you have the `dmalloc' library (-ldmalloc). */
 #cmakedefine H5_HAVE_LIBDMALLOC @H5_HAVE_LIBDMALLOC@
 
+/* Proceed to build with libhdfs */
+#cmakedefine H5_HAVE_LIBHDFS @H5_HAVE_LIBHDFS@
+
+/* Define to 1 if you have the `jvm' library (-ljvm). */
+#cmakedefine H5_HAVE_LIBJVM @H5_HAVE_LIBJVM@
+
 /* Define to 1 if you have the `m' library (-lm). */
 #cmakedefine H5_HAVE_LIBM @H5_HAVE_LIBM@
 
@@ -264,6 +282,15 @@
 /* Define if MPI_Info_c2f and MPI_Info_f2c exists */
 #cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@
 
+/* Define to 1 if you have the <openssl/evp.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@
+
+/* Define to 1 if you have the <openssl/hmac.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_HMAC_H @H5_HAVE_OPENSSL_HMAC_H@
+
+/* Define to 1 if you have the <openssl/sha.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_SHA_H @H5_HAVE_OPENSSL_SHA_H@
+
 /* Define if we have parallel support */
 #cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@
 
@@ -282,6 +309,10 @@
 /* Define to 1 if you have the `rand_r' function. */
 #cmakedefine H5_HAVE_RAND_R @H5_HAVE_RAND_R@
 
+/* Define whether the Read-Only S3 virtual file driver (VFD) should be
+   compiled */
+#cmakedefine H5_HAVE_ROS3_VFD @H5_HAVE_ROS3_VFD@
+
 /* Define to 1 if you have the `round' function. */
 #cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@
 
diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in
index 2117f3b..3451545 100644
--- a/config/cmake/libhdf5.settings.cmake.in
+++ b/config/cmake/libhdf5.settings.cmake.in
@@ -76,6 +76,8 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
           I/O filters (external): @EXTERNAL_FILTERS@
                              MPE: @H5_HAVE_LIBLMPE@
                       Direct VFD: @H5_HAVE_DIRECT@
+              (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@
+            (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@
                          dmalloc: @H5_HAVE_LIBDMALLOC@
   Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@
                      API Tracing: @HDF5_ENABLE_TRACE@
diff --git a/config/cmake_ext_mod/runTest.cmake b/config/cmake_ext_mod/runTest.cmake
index e601653..6f633f3 100644
--- a/config/cmake_ext_mod/runTest.cmake
+++ b/config/cmake_ext_mod/runTest.cmake
@@ -343,6 +343,15 @@ if (TEST_GREP_COMPARE)
   endif ()
 endif ()
 
+# dump the output unless nodisplay option is set
+if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY)
+  file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM)
+  execute_process (
+      COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM}
+      RESULT_VARIABLE TEST_RESULT
+  )
+endif ()
+
 # everything went fine...
 message (STATUS "${TEST_PROGRAM} Passed")
 
diff --git a/config/toolchain/build32.cmake b/config/toolchain/build32.cmake
index d078956..deb5899 100644
--- a/config/toolchain/build32.cmake
+++ b/config/toolchain/build32.cmake
@@ -3,6 +3,42 @@ if (WIN32)
   set (CMAKE_GENERATOR_PLATFORM "x86")
 elseif(APPLE)
   set (CMAKE_OSX_ARCHITECTURES "i386")
+elseif(MINGW)
+  set (CMAKE_SYSTEM_NAME Windows)
+  set (CMAKE_C_COMPILER i686-w64-mingw32-gcc)
+  set (CMAKE_CXX_COMPILER i686-w64-mingw32-g++)
+  set (CMAKE_RC_COMPILER i686-w64-mingw32-windres)
+  set (CMAKE_Fortran_COMPILER i686-w64-mingw32-gfortran)
+
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags")
+  set (CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
+
+  set (LIB32 /usr/lib) # Fedora
+
+  if (EXISTS "/usr/lib32")
+    set (LIB32 /usr/lib32) # Arch, Solus
+  endif ()
+
+  set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE)
+  set (CMAKE_LIBRARY_PATH        ${LIB32} CACHE STRING "library search path"        FORCE)
+
+  # this is probably unlikely to be needed, but just in case
+  set (CMAKE_EXE_LINKER_FLAGS    "-m32 -L${LIB32}" CACHE STRING "executable linker flags"     FORCE)
+  set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE)
+  set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags"         FORCE)
+
+  # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have
+  # to include the regular system .pc files as well (at the end), because some
+  # are not always present in the 32 bit directory
+  if (EXISTS "${LIB32}/pkgconfig")
+    set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig)
+  endif ()
+
+  set (CMAKE_FIND_ROOT_PATH /usr/i686-w64-mingw32)
+  set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+  set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+  set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+  set (CMAKE_CROSSCOMPILING_EMULATOR wine32)
 else ()
   set (CMAKE_SYSTEM_NAME Linux)