Merge branch 'develop' of https://bitbucket.hdfgroup.org/scm/~koziol/hdf5 into develop

6 files changed, 235 insertions, 65 deletions

diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
index 3bd0553..cb1eb48 100644
--- a/config/cmake/ConfigureChecks.cmake
+++ b/config/cmake/ConfigureChecks.cmake
@@ -155,6 +155,21 @@ if (NOT WINDOWS)
 endif ()
 
 #-----------------------------------------------------------------------------
+#  Check if ROS3 driver can be built
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
+  if (HDF5_ENABLE_ROS3_VFD)
+    find_package(CURL REQUIRED)
+    find_package(OpenSSL REQUIRED)
+    if (${CURL_FOUND} AND ${OPENSSL_FOUND})
+      set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
+      list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES})
+    else ()
+      message (STATUS "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
+    endif ()
+endif ()
+
+#-----------------------------------------------------------------------------
 # Check if C has __float128 extension
 #-----------------------------------------------------------------------------
 
diff --git a/config/cmake/FindHDFS.cmake b/config/cmake/FindHDFS.cmake
new file mode 100644
index 0000000..e401a94
--- /dev/null
+++ b/config/cmake/FindHDFS.cmake
@@ -0,0 +1,70 @@
+
+# DerivedFrom: https://github.com/cloudera/Impala/blob/cdh5-trunk/cmake_modules/FindHDFS.cmake
+# - Find HDFS (hdfs.h and libhdfs.so)
+# This module defines
+#  Hadoop_VERSION, version string of ant if found
+#  HDFS_INCLUDE_DIR, directory containing hdfs.h
+#  HDFS_LIBRARIES, location of libhdfs.so
+#  HDFS_FOUND, whether HDFS is found.
+
+exec_program($ENV{HADOOP_HOME}/bin/hadoop ARGS version OUTPUT_VARIABLE Hadoop_VERSION
+             RETURN_VALUE Hadoop_RETURN)
+
+# currently only looking in HADOOP_HOME
+find_path(HDFS_INCLUDE_DIR hdfs.h PATHS
+  $ENV{HADOOP_HOME}/include/
+  # make sure we don't accidentally pick up a different version
+  NO_DEFAULT_PATH
+)
+
+if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8")
+  set(arch_hint "x64")
+elseif ("$ENV{LIB}" MATCHES "(amd64|ia64)")
+  set(arch_hint "x64")
+else ()
+  set(arch_hint "x86")
+endif()
+
+message(STATUS "Architecture: ${arch_hint}")
+
+if ("${arch_hint}" STREQUAL "x64")
+  set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native)
+else ()
+  set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native)
+endif ()
+
+message(STATUS "HDFS_LIB_PATHS: ${HDFS_LIB_PATHS}")
+
+find_library(HDFS_LIB NAMES hdfs PATHS
+  ${HDFS_LIB_PATHS}
+  # make sure we don't accidentally pick up a different version
+  NO_DEFAULT_PATH
+)
+
+if (HDFS_LIB)
+  set(HDFS_FOUND TRUE)
+  set(HDFS_LIBRARIES ${HDFS_LIB})
+  set(HDFS_STATIC_LIB ${HDFS_LIB_PATHS}/${CMAKE_STATIC_LIBRARY_PREFIX}hdfs${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+  add_library(hdfs_static STATIC IMPORTED)
+  set_target_properties(hdfs_static PROPERTIES IMPORTED_LOCATION ${HDFS_STATIC_LIB})
+else ()
+  set(HDFS_FOUND FALSE)
+endif ()
+
+if (HDFS_FOUND)
+  if (NOT HDFS_FIND_QUIETLY)
+    message(STATUS "${Hadoop_VERSION}")
+    message(STATUS "HDFS_INCLUDE_DIR: ${HDFS_INCLUDE_DIR}")
+    message(STATUS "HDFS_LIBRARIES: ${HDFS_LIBRARIES}")
+    message(STATUS "hdfs_static: ${HDFS_STATIC_LIB}")
+  endif ()
+else ()
+  message(FATAL_ERROR "HDFS includes and libraries NOT found."
+    "(${HDFS_INCLUDE_DIR}, ${HDFS_LIB})")
+endif ()
+
+mark_as_advanced(
+  HDFS_LIBRARIES
+  HDFS_INCLUDE_DIR
+)
diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in
index 9e7b8b7..cb05974 100644
--- a/config/cmake/H5pubconf.h.in
+++ b/config/cmake/H5pubconf.h.in
@@ -110,6 +110,9 @@
 /* Define if the function stack tracing code is to be compiled in */
 #cmakedefine H5_HAVE_CODESTACK @H5_HAVE_CODESTACK@
 
+/* Define to 1 if you have the <curl/curl.h> header file. */
+#cmakedefine H5_HAVE_CURL_H @H5_HAVE_CURL_H@
+
 /* Define if Darwin or Mac OS X */
 #cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@
 
@@ -185,6 +188,9 @@
 /* Define to 1 if you have the `gettimeofday' function. */
 #cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@
 
+/* Define to 1 if you have the <hdfs.h> header file. */
+#cmakedefine H5_HAVE_HDFS_H @H5_HAVE_HDFS_H@
+
 /* Define if the compiler understands inline */
 #cmakedefine H5_HAVE_INLINE @H5_HAVE_INLINE@
 
@@ -201,12 +207,24 @@
 /* Define to 1 if you have the <io.h> header file. */
 #cmakedefine H5_HAVE_IO_H @H5_HAVE_IO_H@
 
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#cmakedefine H5_HAVE_LIBCRYPTO @H5_HAVE_LIBCRYPTO@
+
+/* Define to 1 if you have the `curl' library (-lcurl). */
+#cmakedefine H5_HAVE_LIBCURL @H5_HAVE_LIBCURL@
+
 /* Define to 1 if you have the `dl' library (-ldl). */
 #cmakedefine H5_HAVE_LIBDL @H5_HAVE_LIBDL@
 
 /* Define to 1 if you have the `dmalloc' library (-ldmalloc). */
 #cmakedefine H5_HAVE_LIBDMALLOC @H5_HAVE_LIBDMALLOC@
 
+/* Proceed to build with libhdfs */
+#cmakedefine H5_HAVE_LIBHDFS @H5_HAVE_LIBHDFS@
+
+/* Define to 1 if you have the `jvm' library (-ljvm). */
+#cmakedefine H5_HAVE_LIBJVM @H5_HAVE_LIBJVM@
+
 /* Define to 1 if you have the `m' library (-lm). */
 #cmakedefine H5_HAVE_LIBM @H5_HAVE_LIBM@
 
@@ -264,6 +282,15 @@
 /* Define if MPI_Info_c2f and MPI_Info_f2c exists */
 #cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@
 
+/* Define to 1 if you have the <openssl/evp.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@
+
+/* Define to 1 if you have the <openssl/hmac.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_HMAC_H @H5_HAVE_OPENSSL_HMAC_H@
+
+/* Define to 1 if you have the <openssl/sha.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_SHA_H @H5_HAVE_OPENSSL_SHA_H@
+
 /* Define if we have parallel support */
 #cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@
 
@@ -282,6 +309,10 @@
 /* Define to 1 if you have the `rand_r' function. */
 #cmakedefine H5_HAVE_RAND_R @H5_HAVE_RAND_R@
 
+/* Define whether the Read-Only S3 virtual file driver (VFD) should be
+   compiled */
+#cmakedefine H5_HAVE_ROS3_VFD @H5_HAVE_ROS3_VFD@
+
 /* Define to 1 if you have the `round' function. */
 #cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@
 
diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in
index 2117f3b..3451545 100644
--- a/config/cmake/libhdf5.settings.cmake.in
+++ b/config/cmake/libhdf5.settings.cmake.in
@@ -76,6 +76,8 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
           I/O filters (external): @EXTERNAL_FILTERS@
                              MPE: @H5_HAVE_LIBLMPE@
                       Direct VFD: @H5_HAVE_DIRECT@
+              (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@
+            (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@
                          dmalloc: @H5_HAVE_LIBDMALLOC@
   Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@
                      API Tracing: @HDF5_ENABLE_TRACE@
diff --git a/config/conclude_fc.am b/config/conclude_fc.am
deleted file mode 100644
index 15d04a7..0000000
--- a/config/conclude_fc.am
+++ /dev/null
@@ -1,31 +0,0 @@
-## config/conclude_fc.am
-## (Use double hashes for copyright notice so that automake treats it as
-## comments and does not pass it to Makefile.in)
-## Copyright by The HDF Group.
-## Copyright by the Board of Trustees of the University of Illinois.
-## All rights reserved.
-##
-## This file is part of HDF5.  The full HDF5 copyright notice, including
-## terms governing use, modification, and redistribution, is contained in
-## the COPYING file, which can be found at the root of the source code
-## distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
-## If you do not have access to either file, you may request a copy from
-## help@hdfgroup.org.
-
-
-## Textually included at the end of the Fortran HDF5 Makefiles.am.
-
-# Makefile.am include fragment with Fortran helper rules and macros.
-
-# AM_FCCPPFLAGS, FCCPPFLAGS are currently not used.
-
-PPFCCOMPILE = $(FC) $(FCDEFS) $(DEFAULT_INCLUDES)  $(FCINCLUDES) $(AM_FCCPPFLAGS) $(FCCPPFLAGS) $(AM_FCFLAGS) $(FCFLAGS)
-LTPPFCCOMPILE = $(LIBTOOL) --tag=FC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(PPFCCOMPILE)
-
-# Treat all .f90 and .F90 files as preprocessed Fortran.
-.f90.o:
-	$(PPFCCOMPILE) -c -o $@ $<
-.F90.o:
-	$(PPFCCOMPILE) -c -o $@ $<
-
-include $(top_srcdir)/config/conclude.am
diff --git a/config/gnu-fflags b/config/gnu-fflags
index 6b5e6a1..90a3656 100644
--- a/config/gnu-fflags
+++ b/config/gnu-fflags
@@ -1,4 +1,4 @@
-#							-*- shell-script -*-
+#                            -*- shell-script -*-
 #
 # Copyright by The HDF Group.
 # Copyright by the Board of Trustees of the University of Illinois.
@@ -25,9 +25,8 @@
 #
 if test X = "X$f9x_flags_set"; then
     f9x_version="`$FC $FCFLAGS $H5_FCFLAGS -v 2>&1 |grep 'gcc version' |\
-	        sed 's/.*gcc version \([-a-z0-9\.]*\).*/\1/'`"
+            sed 's/.*gcc version \([-a-z0-9\.]*\).*/\1/'`"
     if test X != "X$f9x_version"; then
-#        is_mpi="`$FC $FCFLAGS $H5_FCFLAGS -help 2>&1 |grep 'link MPI'`"
         f9x_vendor=`echo $f9x_version |sed 's/\([a-z]*\).*/\1/'`
         f9x_version=`echo $f9x_version |sed 's/[-a-z]//g'`
         if test X = "X$f9x_vendor" -a X != "X$f9x_version"; then
@@ -37,24 +36,38 @@ if test X = "X$f9x_flags_set"; then
             echo "compiler '$FC' is GNU $f9x_vendor-$f9x_version"
         fi
 
-        # Some version numbers
+        # Get the compiler version numbers
         f9x_vers_major=`echo $f9x_version | cut -f1 -d.`
         f9x_vers_minor=`echo $f9x_version | cut -f2 -d.`
         f9x_vers_patch=`echo $f9x_version | cut -f3 -d.`
         test -n "$f9x_vers_major" || f9x_vers_major=0
         test -n "$f9x_vers_minor" || f9x_vers_minor=0
         test -n "$f9x_vers_patch" || f9x_vers_patch=0
-        f9x_vers_all=`expr $f9x_vers_major '*' 1000000 + $f9x_vers_minor '*' 1000 + $f9x_vers_patch`
     fi
 fi
 
-# Common GNU flags for various situations
 if test "X-gfortran" = "X-$f9x_vendor"; then
-    # Insert section about version specific problems from gnu-flags here, if
-    # necessary.
+
+    FC_BASENAME=gfortran
+    F9XSUFFIXFLAG=""
+    FSEARCH_DIRS=""
+
+    # Need Fortran 2008 support for storage_size() in gcc 4.6 on
+    # (2008ts in some versions)
+    if test $f9x_vers_major -ge 8; then
+        H5_FCFLAGS="$H5_FCFLAGS -std=f2008"
+    elif test $f9x_vers_major -ge 5 -o $f9x_vers_major -eq 4 -a $f9x_vers_minor -ge 6; then
+        H5_FCFLAGS="$H5_FCFLAGS -std=f2008ts"
+    else
+        H5_FCFLAGS="$H5_FCFLAGS -std=f2003"
+    fi
+
+
+    ###############################
+    # Architecture-specific flags #
+    ###############################
 
     arch=
-    # Architecture-specific flags
     # Nothing currently. (Uncomment code below and modify to add any)
     #case "$host_os-$host_cpu" in
     #    *-i686)
@@ -62,43 +75,113 @@ if test "X-gfortran" = "X-$f9x_vendor"; then
     #        ;;
     #esac
 
-    # Host-specific flags
-    # Nothing currently. (Uncomment code below and modify to add any)
-    #case "`hostname`" in
-    #    sleipnir.ncsa.uiuc.edu)
-    #        arch="$arch -pipe"
-    #        ;;
-    #esac
+    H5_FCFLAGS="$H5_FCFLAGS $arch"
 
-    # General
-    FC_BASENAME=gfortran
-    F9XSUFFIXFLAG=""
-    FSEARCH_DIRS=""
-    H5_FCFLAGS="$H5_FCFLAGS -pedantic -Wall -Wextra -Wunderflow -Wimplicit-interface -Wsurprising"
+    ##############
+    # Production #
+    ##############
 
-    # Turn off warnings for passing non-ANSI types to BIND().
-    # We pass a lot of hid_t, etc. types so this generates a LOT of spurious warnings.
-    H5_FCFLAGS="$H5_FCFLAGS -Wno-c-binding-type"
-
-    # Production
     PROD_FCFLAGS=
 
-    # Debug
-    DEBUG_FCFLAGS="-fbounds-check"
+    #########
+    # Debug #
+    #########
+
+    if test $f9x_vers_major -ge 5 -o $f9x_vers_major -eq 4 -a $f9x_vers_minor -ge 5; then
+        DEBUG_FCFLAGS="-fcheck=all"
+    else
+        DEBUG_FCFLAGS="-fbounds-check"
+    fi
+
+    ###########
+    # Symbols #
+    ###########
 
-    # Symbols
     SYMBOLS_FCFLAGS="-g"
     NO_SYMBOLS_FCFLAGS="-s"
 
-    # Profiling
+    #############
+    # Profiling #
+    #############
+
     PROFILE_FCFLAGS="-pg"
 
-    # Optimization
-    HIGH_OPT_FCFLAGS="-O2"
-    DEBUG_OPT_FCFLAGS="-O0"
+    ################
+    # Optimization #
+    ################
+
+    if test $f9x_vers_major -le 4; then
+        HIGH_OPT_CFLAGS="-O3"
+        DEBUG_OPT_CFLAGS=
+    else
+        HIGH_OPT_CFLAGS="-O3"
+        DEBUG_OPT_CFLAGS="-Og"
+    fi
     NO_OPT_FCFLAGS="-O0"
 
-    # Flags are set
+    ############
+    # Warnings #
+    ############
+
+    ###########
+    # General #
+    ###########
+
+    H5_FCFLAGS="$H5_FCFLAGS -pedantic -Wall -Wextra -Wunderflow -Wimplicit-interface -Wsurprising"
+    H5_FCFLAGS="$H5_FCFLAGS -Waliasing -Wcharacter-truncation -Wimplicit-procedure"
+
+    #############################
+    # Version-specific warnings #
+    #############################
+
+    # gfortran 4.3 (nothing new)
+
+    # gfortran 4.4
+    if test $f9x_vers_major -ge 5 -o $f9x_vers_major -eq 4 -a $f9x_vers_minor -ge 4; then
+        H5_FCFLAGS="$H5_FCFLAGS -Warray-temporaries -Wintrinsics-std"
+    fi
+
+    # gfortran 4.5 (nothing new)
+
+    # gfortran 4.6 (nothing new)
+
+    # gfortran 4.7
+    if test $f9x_vers_major -ge 5 -o $f9x_vers_major -eq 4 -a $f9x_vers_minor -ge 7; then
+        H5_FCFLAGS="$H5_FCFLAGS -Wreal-q-constant -Wfunction-elimination"
+    fi
+
+    # gfortran 4.8
+    if test $f9x_vers_major -ge 5 -o $f9x_vers_major -eq 4 -a $f9x_vers_minor -ge 8; then
+        H5_FCFLAGS="$H5_FCFLAGS -Wrealloc-lhs -Wrealloc-lhs-all"
+        # Turn off warnings for passing non-ANSI types to BIND().
+        # We pass a lot of hid_t, etc. types so this generates a LOT of spurious warnings.
+        H5_FCFLAGS="$H5_FCFLAGS -Wno-c-binding-type"
+    fi
+
+    # gfortran 4.9 (nothing new)
+
+    # gfortran 5
+    if test $f9x_vers_major -ge 5; then
+        H5_FCFLAGS="$H5_FCFLAGS -Wuse-without-only"
+    fi
+
+    # gfortran 6
+    if test $f9x_vers_major -ge 6; then
+        H5_FCFLAGS="$H5_FCFLAGS -Winteger-division"
+    fi
+
+    # gfortran 7 (nothing new)
+
+    # gfortran 8
+    if test $f9x_vers_major -ge 8; then
+        H5_FCFLAGS="$H5_FCFLAGS -Wfrontend-loop-interchange"
+    fi
+
+    # gfortran 9 (nothing new)
+
+    #################
+    # Flags are set #
+    #################
     f9x_flags_set=yes
 fi