From efe7cb72ba123177c4441720bb8a9066a0b1feb0 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:47:55 -0600 Subject: Add binary build for linux S3 (#3936) * Reorg and update options for doc and cmake config * Correct subfiling depends * Add linux S3 binaries to GH published files * Correct path * Try reducing parallel jobs in tests * Change examples jobs for test from 8 to 4 --- .github/workflows/cmake-ctest.yml | 64 ++++++++++++++++++++++ .github/workflows/release-files.yml | 8 +++ CMakePresets.json | 46 ++++++++++++++++ HDF5Examples/C/CMakeLists.txt | 2 +- .../config/cmake-presets/hidden-presets.json | 2 +- config/cmake-presets/hidden-presets.json | 2 +- config/cmake/scripts/CTestScript.cmake | 2 +- 7 files changed, 122 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake-ctest.yml b/.github/workflows/cmake-ctest.yml index cc8bdaf..aa13922 100644 --- a/.github/workflows/cmake-ctest.yml +++ b/.github/workflows/cmake-ctest.yml @@ -230,6 +230,70 @@ jobs: path: ${{ runner.workspace }}/build/${{ steps.set-file-base.outputs.FILE_BASE }}-osx12.tar.gz if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` + build_and_test_S3_linux: + # Linux S3 (Ubuntu) w/ gcc + CMake + # + name: "Ubuntu gcc CMake S3" + runs-on: ubuntu-latest + steps: + - name: Install CMake Dependencies (Linux S3) + run: | + sudo apt-get install ninja-build doxygen graphviz + sudo apt install libssl3 libssl-dev libcurl4 libcurl4-openssl-dev + + - name: Set file base name (Linux S3) + id: set-file-base + run: | + FILE_NAME_BASE=$(echo "${{ inputs.file_base }}") + echo "FILE_BASE=$FILE_NAME_BASE" >> $GITHUB_OUTPUT + + # Get files created by release script + - name: Get tgz-tarball (Linux S3) + uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 + with: + name: tgz-tarball + path: ${{ github.workspace }} + + - name: List files for the space (Linux S3) + run: | + ls -l ${{ github.workspace }} + ls ${{ runner.workspace }} + + - name: Uncompress source (Linux S3) + run: tar -zxvf ${{ github.workspace }}/${{ steps.set-file-base.outputs.FILE_BASE }}.tar.gz + + - name: Run ctest (Linux S3) + run: | + cd "${{ runner.workspace }}/hdf5/hdfsrc" + cmake --workflow --preset=ci-StdShar-GNUC-S3 --fresh + shell: bash + + - name: Publish binary (Linux S3) + id: publish-ctest-binary + run: | + mkdir "${{ runner.workspace }}/build" + mkdir "${{ runner.workspace }}/build/hdf5" + cp ${{ runner.workspace }}/hdf5/hdfsrc/COPYING ${{ runner.workspace }}/build/hdf5 + cp ${{ runner.workspace }}/hdf5/hdfsrc/COPYING_LBNL_HDF5 ${{ runner.workspace }}/build/hdf5 + cp ${{ runner.workspace }}/hdf5/hdfsrc/README.md ${{ runner.workspace }}/build/hdf5 + cp ${{ runner.workspace }}/hdf5/build/ci-StdShar-GNUC-S3/*.tar.gz ${{ runner.workspace }}/build/hdf5 + cd "${{ runner.workspace }}/build" + tar -zcvf ${{ steps.set-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz hdf5 + shell: bash + + - name: List files in the space (Linux S3) + run: | + ls ${{ github.workspace }} + ls -l ${{ runner.workspace }} + + # Save files created by ctest script + - name: Save published binary (Linux S3) + uses: actions/upload-artifact@v4 + with: + name: tgz-ubuntu-2204_gcc_s3-binary + path: ${{ runner.workspace }}/build/${{ steps.set-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz + if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` + ####### intel builds build_and_test_win_intel: # Windows w/ OneAPI + CMake diff --git a/.github/workflows/release-files.yml b/.github/workflows/release-files.yml index aaeebd4..6e9b60e 100644 --- a/.github/workflows/release-files.yml +++ b/.github/workflows/release-files.yml @@ -114,6 +114,12 @@ jobs: name: tgz-ubuntu-2204_gcc-binary path: ${{ github.workspace }} + - name: Get published binary (Linux S3) + uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 + with: + name: tgz-ubuntu-2204_gcc_s3-binary + path: ${{ github.workspace }} + - name: Get published binary (Windows_intel) uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 with: @@ -144,6 +150,7 @@ jobs: ${{ steps.get-file-base.outputs.FILE_BASE }}.zip ${{ steps.get-file-base.outputs.FILE_BASE }}-osx12.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc.tar.gz + ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_cl.zip ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_intel.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_intel.zip @@ -163,6 +170,7 @@ jobs: ${{ steps.get-file-base.outputs.FILE_BASE }}.zip ${{ steps.get-file-base.outputs.FILE_BASE }}-osx12.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc.tar.gz + ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_cl.zip ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_intel.tar.gz ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_intel.zip diff --git a/CMakePresets.json b/CMakePresets.json index 84b4f2f..65c56d7 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -94,6 +94,14 @@ } }, { + "name": "ci-S3", + "hidden": true, + "cacheVariables": { + "HDF5_ENABLE_ROS3_VFD": "ON", + "HDF5_ENABLE_HDFS": "OFF" + } + }, + { "name": "ci-StdShar", "hidden": true, "inherits": ["ci-StdCompression", "ci-StdExamples", "ci-StdPlugins"], @@ -150,6 +158,14 @@ ] }, { + "name": "ci-StdShar-GNUC-S3", + "description": "GNUC S3 Config for x64 (Release)", + "inherits": [ + "ci-StdShar-GNUC", + "ci-S3" + ] + }, + { "name": "ci-StdShar-Intel", "description": "Intel Standard Config for x64 (Release)", "inherits": [ @@ -188,6 +204,15 @@ ] }, { + "name": "ci-StdShar-GNUC-S3", + "description": "GNUC S3 Build for x64 (Release)", + "configurePreset": "ci-StdShar-GNUC-S3", + "verbose": true, + "inherits": [ + "ci-x64-Release-GNUC" + ] + }, + { "name": "ci-StdShar-Intel", "description": "Intel Standard Build for x64 (Release)", "configurePreset": "ci-StdShar-Intel", @@ -225,6 +250,13 @@ ] }, { + "name": "ci-StdShar-GNUC-S3", + "configurePreset": "ci-StdShar-GNUC-S3", + "inherits": [ + "ci-x64-Release-GNUC" + ] + }, + { "name": "ci-StdShar-Intel", "configurePreset": "ci-StdShar-Intel", "inherits": [ @@ -254,6 +286,11 @@ "inherits": "ci-x64-Release-GNUC" }, { + "name": "ci-StdShar-GNUC-S3", + "configurePreset": "ci-StdShar-GNUC-S3", + "inherits": "ci-x64-Release-GNUC" + }, + { "name": "ci-StdShar-Intel", "configurePreset": "ci-StdShar-Intel", "inherits": "ci-x64-Release-Intel" @@ -288,6 +325,15 @@ ] }, { + "name": "ci-StdShar-GNUC-S3", + "steps": [ + {"type": "configure", "name": "ci-StdShar-GNUC-S3"}, + {"type": "build", "name": "ci-StdShar-GNUC-S3"}, + {"type": "test", "name": "ci-StdShar-GNUC-S3"}, + {"type": "package", "name": "ci-StdShar-GNUC-S3"} + ] + }, + { "name": "ci-StdShar-Intel", "steps": [ {"type": "configure", "name": "ci-StdShar-Intel"}, diff --git a/HDF5Examples/C/CMakeLists.txt b/HDF5Examples/C/CMakeLists.txt index 4e589bc..4ac8574 100644 --- a/HDF5Examples/C/CMakeLists.txt +++ b/HDF5Examples/C/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 3.12) -PROJECT (HDF5Examples_C) +project (HDF5Examples_C) #----------------------------------------------------------------------------- # Build the C Examples diff --git a/HDF5Examples/config/cmake-presets/hidden-presets.json b/HDF5Examples/config/cmake-presets/hidden-presets.json index 883b903..8b7f71b 100644 --- a/HDF5Examples/config/cmake-presets/hidden-presets.json +++ b/HDF5Examples/config/cmake-presets/hidden-presets.json @@ -274,7 +274,7 @@ "execution": { "noTestsAction": "error", "timeout": 600, - "jobs": 8 + "jobs": 4 } }, { diff --git a/config/cmake-presets/hidden-presets.json b/config/cmake-presets/hidden-presets.json index 24ab42e..2249651 100644 --- a/config/cmake-presets/hidden-presets.json +++ b/config/cmake-presets/hidden-presets.json @@ -400,7 +400,7 @@ "execution": { "noTestsAction": "error", "timeout": 600, - "jobs": 8 + "jobs": 4 } }, { diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake index 37bf0d4..4f7eb4b 100644 --- a/config/cmake/scripts/CTestScript.cmake +++ b/config/cmake/scripts/CTestScript.cmake @@ -81,7 +81,7 @@ if (CTEST_USE_TAR_SOURCE) ## Uncompress source if tar file provided ## -------------------------- if (WIN32 AND NOT MINGW) - message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} x ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip]") + message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip]") execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip RESULT_VARIABLE rv) else () message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}/${CTEST_USE_TAR_SOURCE}.tar]") -- cgit v0.12