From 3fed3735dba9922e9b0ced19e995039bebd9c5d1 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Tue, 17 Sep 2019 14:41:00 -0500 Subject: HDFFV-10903 merge updated changes --- config/cmake/FindHDFS.cmake | 70 +++++++++++++++++++++++++++++++ config/cmake/HDFCompilerFlags.cmake | 76 +++++++++++++++++++++++++++++++++- config/cmake/libhdf5.settings.cmake.in | 8 ++-- java/src/jni/h5Constants.c | 16 ++++++- java/src/jni/h5fImp.c | 4 -- src/CMakeLists.txt | 2 - tools/lib/h5tools_dump.c | 8 ++++ tools/test/h5copy/testh5copy.sh.in | 9 ++-- tools/test/h5repack/h5repack.sh.in | 9 +++- 9 files changed, 182 insertions(+), 20 deletions(-) create mode 100644 config/cmake/FindHDFS.cmake diff --git a/config/cmake/FindHDFS.cmake b/config/cmake/FindHDFS.cmake new file mode 100644 index 0000000..e401a94 --- /dev/null +++ b/config/cmake/FindHDFS.cmake @@ -0,0 +1,70 @@ + +# DerivedFrom: https://github.com/cloudera/Impala/blob/cdh5-trunk/cmake_modules/FindHDFS.cmake +# - Find HDFS (hdfs.h and libhdfs.so) +# This module defines +# Hadoop_VERSION, version string of ant if found +# HDFS_INCLUDE_DIR, directory containing hdfs.h +# HDFS_LIBRARIES, location of libhdfs.so +# HDFS_FOUND, whether HDFS is found. + +exec_program($ENV{HADOOP_HOME}/bin/hadoop ARGS version OUTPUT_VARIABLE Hadoop_VERSION + RETURN_VALUE Hadoop_RETURN) + +# currently only looking in HADOOP_HOME +find_path(HDFS_INCLUDE_DIR hdfs.h PATHS + $ENV{HADOOP_HOME}/include/ + # make sure we don't accidentally pick up a different version + NO_DEFAULT_PATH +) + +if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8") + set(arch_hint "x64") +elseif ("$ENV{LIB}" MATCHES "(amd64|ia64)") + set(arch_hint "x64") +else () + set(arch_hint "x86") +endif() + +message(STATUS "Architecture: ${arch_hint}") + +if ("${arch_hint}" STREQUAL "x64") + set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native) +else () + set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native) +endif () + +message(STATUS "HDFS_LIB_PATHS: ${HDFS_LIB_PATHS}") + +find_library(HDFS_LIB NAMES hdfs PATHS + ${HDFS_LIB_PATHS} + # make sure we don't accidentally pick up a different version + NO_DEFAULT_PATH +) + +if (HDFS_LIB) + set(HDFS_FOUND TRUE) + set(HDFS_LIBRARIES ${HDFS_LIB}) + set(HDFS_STATIC_LIB ${HDFS_LIB_PATHS}/${CMAKE_STATIC_LIBRARY_PREFIX}hdfs${CMAKE_STATIC_LIBRARY_SUFFIX}) + + add_library(hdfs_static STATIC IMPORTED) + set_target_properties(hdfs_static PROPERTIES IMPORTED_LOCATION ${HDFS_STATIC_LIB}) +else () + set(HDFS_FOUND FALSE) +endif () + +if (HDFS_FOUND) + if (NOT HDFS_FIND_QUIETLY) + message(STATUS "${Hadoop_VERSION}") + message(STATUS "HDFS_INCLUDE_DIR: ${HDFS_INCLUDE_DIR}") + message(STATUS "HDFS_LIBRARIES: ${HDFS_LIBRARIES}") + message(STATUS "hdfs_static: ${HDFS_STATIC_LIB}") + endif () +else () + message(FATAL_ERROR "HDFS includes and libraries NOT found." + "(${HDFS_INCLUDE_DIR}, ${HDFS_LIB})") +endif () + +mark_as_advanced( + HDFS_LIBRARIES + HDFS_INCLUDE_DIR +) diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index aea417a..37ddcbb 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -212,7 +212,8 @@ if (NOT MSVC AND CMAKE_COMPILER_IS_GNUCC) if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) set (H5_CFLAGS4 "${H5_CFLAGS4} -Wattribute-alias=2 -Wmissing-profile") endif () - +elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Minform=inform") endif () #----------------------------------------------------------------------------- @@ -339,3 +340,76 @@ endif () if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") endif () + +#----------------------------------------------------------------------------- +# Option for --enable-asserts +# By default, CMake adds NDEBUG to CMAKE_${lang}_FLAGS for Release build types +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_ASSERTS "Determines whether NDEBUG is defined to control assertions." OFF) +if (HDF5_ENABLE_ASSERTS) + add_compile_options ("-UNDEBUG") +else () + add_compile_options ("-DNDEBUG") +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_ASSERTS) + +#----------------------------------------------------------------------------- +# Option for --enable-symbols +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_SYMBOLS "Add debug symbols to the library independent of the build mode and optimization level." OFF) +if (HDF5_ENABLE_SYMBOLS) + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fno-omit-frame-pointer") + endif () + if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") + endif () + endif () +else () + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s") + endif () + if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set (CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s") + endif () + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_SYMBOLS) + +#----------------------------------------------------------------------------- +# Option for --enable-profiling +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_PROFILING "Enable profiling flags independently from the build mode." OFF) +if (HDF5_ENABLE_PROFILING) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PROFILE_CFLAGS}") + if(CMAKE_CXX_COMPILER_LOADED) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PROFILE_CXXFLAGS}") + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_PROFILING) + +#----------------------------------------------------------------------------- +# Option for --enable-optimization +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_OPTIMIZATION "Enable optimization flags/settings independently from the build mode" OFF) +if (HDF5_ENABLE_OPTIMIZATION) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OPTIMIZE_CFLAGS}") + if(CMAKE_CXX_COMPILER_LOADED) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTIMIZE_CXXFLAGS}") + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_OPTIMIZATION) diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index 3451545..8397d68 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -14,10 +14,10 @@ General Information: Compiling Options: ------------------ Build Mode: @CMAKE_BUILD_TYPE@ - Debugging Symbols: @SYMBOLS@ - Asserts: @ASSERTS@ - Profiling: @PROFILING@ - Optimization Level: @OPTIMIZATION@ + Debugging Symbols: @HDF5_ENABLE_SYMBOLS@ + Asserts: @HDF5_ENABLE_ASSERTS@ + Profiling: @HDF5_ENABLE_PROFILING@ + Optimization Level: @HDF5_ENABLE_OPTIMIZATION@ Linking Options: ---------------- diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index f1bd951..f9b0cfd 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -438,7 +438,13 @@ Java_hdf_hdf5lib_HDF5Constants_H5FD_1DIRECT(JNIEnv *env, jclass cls) { JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1FAMILY(JNIEnv *env, jclass cls) { return H5FD_FAMILY; } JNIEXPORT jlong JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5FD_1HDFS(JNIEnv *env, jclass cls) { return H5FD_HDFS; } +Java_hdf_hdf5lib_HDF5Constants_H5FD_1HDFS(JNIEnv *env, jclass cls) { +#ifdef H5_HAVE_LIBHDFS + return H5FD_HDFS; +#else + return -1; +#endif +} JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1LOG(JNIEnv *env, jclass cls) { return H5FD_LOG; } JNIEXPORT jlong JNICALL @@ -448,7 +454,13 @@ Java_hdf_hdf5lib_HDF5Constants_H5FD_1MULTI(JNIEnv *env, jclass cls) { return H5F JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1SEC2(JNIEnv *env, jclass cls) { return H5FD_SEC2; } JNIEXPORT jlong JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5FD_1ROS3(JNIEnv *env, jclass cls) { return H5FD_ROS3; } +Java_hdf_hdf5lib_HDF5Constants_H5FD_1ROS3(JNIEnv *env, jclass cls) { +#ifdef H5_HAVE_ROS3_VFD + return H5FD_ROS3; +#else + return -1; +#endif +} JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1STDIO(JNIEnv *env, jclass cls) { return H5FD_STDIO; } JNIEXPORT jlong JNICALL diff --git a/java/src/jni/h5fImp.c b/java/src/jni/h5fImp.c index 2924d8f..115e8f4 100644 --- a/java/src/jni/h5fImp.c +++ b/java/src/jni/h5fImp.c @@ -161,10 +161,6 @@ Java_hdf_hdf5lib_H5_H5Fis_1hdf5 UNUSED(clss); -#ifdef H5_NO_DEPRECATED_SYMBOLS - H5_UNIMPLEMENTED(ENVONLY, "H5Fis_hdf5: not implemented"); -#endif - if (NULL == name) H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Fis_hdf5: file name is NULL"); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a3f98ed..fe93d54 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1079,7 +1079,6 @@ if (NOT ONLY_SHARED_LIBS) PUBLIC ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $,DEBUG,NDEBUG> PRIVATE $<$:H5_DEBUG_API> # Enable tracing of the API $<$:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> @@ -1117,7 +1116,6 @@ if (BUILD_SHARED_LIBS) "H5_BUILT_AS_DYNAMIC_LIB" ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $,DEBUG,NDEBUG> PRIVATE $<$:H5_HAVE_THREADSAFE> $<$:H5_DEBUG_API> # Enable tracing of the API diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index 8b36280..beb397c 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -2289,6 +2289,14 @@ h5tools_print_datatype(FILE *stream, h5tools_str_t *buffer, const h5tool_format_ h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); H5free_memory(ttag); + + if((size = H5Tget_size(type)) <= 0) { + ctx->need_prefix = TRUE; + + h5tools_str_reset(buffer); + h5tools_str_append(buffer, "OPAQUE_SIZE \"%s\";", size); + h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); + } } ctx->indent_level--; diff --git a/tools/test/h5copy/testh5copy.sh.in b/tools/test/h5copy/testh5copy.sh.in index 50758b5..2440ca4 100644 --- a/tools/test/h5copy/testh5copy.sh.in +++ b/tools/test/h5copy/testh5copy.sh.in @@ -504,22 +504,21 @@ fi TOOLTEST -i $TESTFILE -o $TESTDIR/dsrename.out.h5 -v -s compound -d rename echo "Test copying empty, 'full' & 'nested' groups" -if test $USE_FILTER_DEFLATE = "yes" ; then TOOLTEST -i $TESTFILE -o $TESTDIR/grp_empty.out.h5 -v -s grp_empty -d grp_empty -fi +if test $USE_FILTER_DEFLATE = "yes" ; then TOOLTEST -i $TESTFILE -o $TESTDIR/grp_dsets.out.h5 -v -s grp_dsets -d grp_dsets TOOLTEST -i $TESTFILE -o $TESTDIR/grp_nested.out.h5 -v -s grp_nested -d grp_nested +fi TOOLTEST -i $TESTFILE -o $TESTDIR/grp_attr.out.h5 -v -s grp_attr -d grp_attr -if test $USE_FILTER_DEFLATE = "yes" ; then echo "Test copying dataset within group in source file to group in destination" TOOLTEST_PREFILL -i $TESTFILE -o $TESTDIR/simple_group.out.h5 grp_dsets grp_dsets /grp_dsets/simple /grp_dsets/simple_group +if test $USE_FILTER_DEFLATE = "yes" ; then echo "Test copying & renaming group" TOOLTEST -i $TESTFILE -o $TESTDIR/grp_rename.out.h5 -v -s grp_dsets -d grp_rename -fi - echo "Test copying 'full' group hierarchy into group in destination file" TOOLTEST_PREFILL -i $TESTFILE -o $TESTDIR/grp_dsets_rename.out.h5 grp_dsets grp_rename grp_dsets /grp_rename/grp_dsets +fi echo "Test copying objects into group hier. that doesn't exist yet in destination file" TOOLTEST -i $TESTFILE -o $TESTDIR/A_B1_simple.out.h5 -vp -s simple -d /A/B1/simple diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index a36eb08..1586485 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -1387,7 +1387,12 @@ TOOLTEST add_alignment $arg TOOLTEST upgrade_layout h5repack_layouto.h5 # test for datum size > H5TOOLS_MALLOCSIZE -TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1 +arg="h5repack_objs.h5 -f GZIP=1" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST gt_mallocsize $arg +fi # Check repacking file with committed datatypes in odd configurations TOOLTEST committed_dt h5repack_named_dtypes.h5 @@ -1458,7 +1463,7 @@ VERIFY_SUPERBLOCK 1 2 2 h5repack_layout.h5 -j 1 -k 2 h5repack_layout.h5 # -j 2 -k 2, superblock will be 3 VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5 # -j 0 -k 1, file cannot be opened -VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 +VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR -- cgit v0.12