diff options
Diffstat (limited to '.github')
-rw-r--r-- | .github/workflows/vol.yml | 48 | ||||
-rw-r--r-- | .github/workflows/vol_adios2.yml | 119 | ||||
-rw-r--r-- | .github/workflows/vol_async.yml | 94 | ||||
-rw-r--r-- | .github/workflows/vol_cache.yml | 164 | ||||
-rw-r--r-- | .github/workflows/vol_ext_passthru.yml | 64 | ||||
-rw-r--r-- | .github/workflows/vol_log.yml | 89 | ||||
-rw-r--r-- | .github/workflows/vol_rest.yml | 195 |
7 files changed, 773 insertions, 0 deletions
diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml new file mode 100644 index 0000000..ac2a9f5 --- /dev/null +++ b/.github/workflows/vol.yml @@ -0,0 +1,48 @@ +name: hdf5 VOL connectors CI + +# Run VOL connector CI daily at 06:00 CDT (11:00 UTC) +on: + workflow_dispatch: + schedule: + - cron: "0 11 * * *" + +permissions: + contents: read + +jobs: + # Build and test individual VOL connectors by using HDF5's + # CMake FetchContent functionality. + #hdf5_vol_daos_fetchcontent: + # uses: ./.github/workflows/vol_daos.yml + # with: + # build_mode: "Release" + + hdf5_vol_rest_fetchcontent: + uses: ./.github/workflows/vol_rest.yml + with: + build_mode: "Release" + + hdf5_vol_ext_passthru_fetchcontent: + uses: ./.github/workflows/vol_ext_passthru.yml + with: + build_mode: "Release" + + hdf5_vol_async_fetchcontent: + uses: ./.github/workflows/vol_async.yml + with: + build_mode: "Release" + + hdf5_vol_cache_fetchcontent: + uses: ./.github/workflows/vol_cache.yml + with: + build_mode: "Release" + + hdf5_vol_adios2: + uses: ./.github/workflows/vol_adios2.yml + with: + build_mode: "Release" + + hdf5_vol_log: + uses: ./.github/workflows/vol_log.yml + with: + build_mode: "Release" diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml new file mode 100644 index 0000000..35fde5e --- /dev/null +++ b/.github/workflows/vol_adios2.yml @@ -0,0 +1,119 @@ +name: Test HDF5 ADIOS2 VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +env: + ADIOS2_COMMIT: 3adf20a929b69c23312a6b5f3cccc49376df77e8 + ADIOS2_COMMIT_SHORT: 3adf20a + +jobs: + build_and_test: + name: Test HDF5 ADIOS2 VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + - name: Configure HDF5 + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + echo "PATH=${{ runner.workspace }}/hdf5_build/bin:${PATH}" >> $GITHUB_ENV + + # Since the HDF5 ADIOS2 VOL connector is part of the ADIOS2 repository, + # it is difficult to use CMake's FetchContent functionality to fetch + # and build the ADIOS2 connector. Also, since building of ADIOS2 takes + # a while, it isn't ideal to have to rebuild it every time we want to + # test against changes in HDF5 or the VOL connector. Therefore, just + # use a fixed commit for the build of ADIOS2 so we can cache that and + # still test the connector against changes in HDF5. + - name: Restore ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) installation cache + id: cache-adios2 + uses: actions/cache@v3 + with: + path: ${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install + key: ${{ runner.os }}-${{ runner.arch }}-adios2-${{ env.ADIOS2_COMMIT }}-${{ inputs.build_mode }}-cache + + - if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + name: Checkout ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) + uses: actions/checkout@v4 + with: + repository: ornladios/ADIOS2 + ref: ${{ env.ADIOS2_COMMIT }} + path: adios2 + + - if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + name: Install ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) + env: + CXX: mpic++ + CC: mpicc + run: | + mkdir adios2/build + cd adios2/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install \ + -DADIOS2_USE_HDF5:BOOL=ON \ + -DHDF5_ROOT=${{ runner.workspace }}/hdf5_build/ \ + .. + make -j2 + make -j2 install + + - name: Cache ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) installation + uses: actions/cache/save@v3 + if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + with: + path: ${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install + key: ${{ runner.os }}-${{ runner.arch }}-adios2-${{ env.ADIOS2_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Set environment variables for tests + run: | + echo "HDF5_PLUGIN_PATH=${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install/lib" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=ADIOS2_VOL" >> $GITHUB_ENV + + # Skip parallel testing for now as it appears to hang + - name: Test HDF5 ADIOS2 VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the ADIOS2 VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml new file mode 100644 index 0000000..bb4c3a1 --- /dev/null +++ b/.github/workflows/vol_async.yml @@ -0,0 +1,94 @@ +name: Test HDF5 async VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 asynchronous I/O VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + - name: Checkout Argobots + uses: actions/checkout@v4 + with: + repository: pmodels/argobots + path: abt + + # Argobots builds and installs fairly quickly, + # so no caching is currently performed here + - name: Install Argobots + working-directory: ${{ github.workspace }}/abt + run: | + ./autogen.sh + ./configure --prefix=/usr/local + make -j2 + sudo make -j2 install + + - name: Configure HDF5 with asynchronous I/O VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ + -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and asynchronous I/O VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + # Workaround for asynchronous I/O VOL CMake issue + - name: Copy testing files + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cp bin/async_test* ./_deps/vol-async-build/test + + - name: Test HDF5 asynchronous I/O VOL connector with external tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "async_test" . + + - name: Test HDF5 asynchronous I/O VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Async VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-async" . diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml new file mode 100644 index 0000000..1a8c40c --- /dev/null +++ b/.github/workflows/vol_cache.yml @@ -0,0 +1,164 @@ +name: Test HDF5 cache VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + strategy: + matrix: + name: + - "Test HDF5 cache VOL connector" + - "Test HDF5 cache VOL connector atop async VOL connector" + async: [false, true] + exclude: + - name: "Test HDF5 cache VOL connector" + async: true + + - name: "Test HDF5 cache VOL connector atop async VOL connector" + async: false + + + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + - name: Checkout Argobots + uses: actions/checkout@v4 + with: + repository: pmodels/argobots + path: abt + + # Argobots builds and installs fairly quickly, + # so no caching is currently performed here + - name: Install Argobots + working-directory: ${{ github.workspace }}/abt + run: | + ./autogen.sh + ./configure --prefix=/usr/local + make -j2 + sudo make -j2 install + + - name: Set environment variables for configuration (cache VOL only) + run: | + echo "HDF5_VOL_CACHE_TEST_NAME=cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=0\;under_info={}\;" >> $GITHUB_ENV + if: ${{ ! matrix.async }} + + - name: Set environment variables for configuration (cache VOL atop async VOL) + run: | + echo "HDF5_VOL_CACHE_TEST_NAME=cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=512\;under_info={under_vol=0\;under_info={}}\;" >> $GITHUB_ENV + if: ${{ matrix.async }} + + # Define ASYNC_INCLUDE_DIR, ASYNC_INCLUDE_DIRS and ASYNC_LIBRARIES to + # patch around having the cache VOL find the async VOL when they're built + # at the same time. Once the Async and Cache VOLs create CMake .config + # files, this should no longer be needed with CMake 3.24 and newer (see + # FetchContent's OVERRIDE_FIND_PACKAGE) + - name: Configure HDF5 with cache VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ + -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ + -DHDF5_VOL_URL02:STRING="https://github.com/HDFGroup/vol-cache.git" \ + -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-CACHE_NAME:STRING="$HDF5_VOL_CACHE_TEST_NAME" \ + -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ + -DASYNC_INCLUDE_DIR=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_INCLUDE_DIRS=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_LIBRARIES=${{ github.workspace }}/hdf5/build/bin/libasynchdf5.a\;${{ github.workspace }}/hdf5/build/bin/libh5async.so \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and cache VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + - name: Create cache VOL connector configuration file for testing + shell: bash + run: | + mkdir -p $GITHUB_WORKSPACE/scratch + touch $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_SCOPE: LOCAL" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_PATH: $GITHUB_WORKSPACE/scratch" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_SIZE: 4294967296" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config1.cfg + + # Workaround for cache VOL CMake issue + - name: Copy testing files + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cp bin/test_file.exe ./_deps/vol-cache-build/tests + cp bin/test_group.exe ./_deps/vol-cache-build/tests + cp bin/test_dataset.exe ./_deps/vol-cache-build/tests + cp bin/test_dataset_async_api.exe ./_deps/vol-cache-build/tests + cp bin/test_write_multi.exe ./_deps/vol-cache-build/tests + cp bin/test_multdset.exe ./_deps/vol-cache-build/tests + + - name: Set environment variables for external tests (cache VOL only) + run: | + echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config1.cfg;under_vol=0;under_info={};" >> $GITHUB_ENV + if: ${{ ! matrix.async }} + + - name: Set environment variables for external tests (cache VOL atop async VOL) + run: | + echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config1.cfg;under_vol=512;under_info={under_vol=0;under_info={}};" >> $GITHUB_ENV + if: ${{ matrix.async }} + + # Until cache VOL tests are namespaced properly, run them directly + - name: Test HDF5 cache VOL connector with external tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_file$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_group$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset_async_api$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_write_multi$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_multdset$" . + + - name: Test HDF5 cache VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Cache VOL connector with the HDF5 API tests yet + # when it's stacked on top of the Async connector, as it doesn't + # currently pass all the tests due to the Async connector not passing + # all the tests. Leave the step in, but skip it to leave an indication + # that this should be re-enabled in the future. + if: ${{ ! matrix.async }} + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . diff --git a/.github/workflows/vol_ext_passthru.yml b/.github/workflows/vol_ext_passthru.yml new file mode 100644 index 0000000..337130b --- /dev/null +++ b/.github/workflows/vol_ext_passthru.yml @@ -0,0 +1,64 @@ +name: Test HDF5 external pass-through VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 external passthrough VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + - name: Checkout vol-external-passthrough + uses: actions/checkout@v4 + with: + repository: hpc-io/vol-external-passthrough + path: vol-external-passthrough + + - name: Configure HDF5 with external passthrough VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-external-passthrough.git" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_NAME:STRING="pass_through_ext under_vol=0\;under_info={}\;" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and external passthrough VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + + - name: Test HDF5 external passthrough VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-external-passthrough" . diff --git a/.github/workflows/vol_log.yml b/.github/workflows/vol_log.yml new file mode 100644 index 0000000..0a35578 --- /dev/null +++ b/.github/workflows/vol_log.yml @@ -0,0 +1,89 @@ +name: Test HDF5 Log-based VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 Log-based VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev zlib1g-dev + #mpich + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + # Log-based VOL currently doesn't have CMake support + - name: Configure HDF5 + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + echo "PATH=${{ runner.workspace }}/hdf5_build/bin:${PATH}" >> $GITHUB_ENV + + - name: Checkout Log-based VOL + uses: actions/checkout@v4 + with: + repository: DataLib-ECP/vol-log-based + path: vol-log-based + + - name: Build HDF5 Log-based VOL connector and test with external tests + env: + CXX: mpic++ + CC: mpicc + LD_LIBRARY_PATH: ${{ runner.workspace }}/hdf5_build/lib + run: | + cd vol-log-based + autoreconf -i + ./configure --prefix=${{ runner.workspace }}/vol-log-based-build --with-hdf5=${{ runner.workspace }}/hdf5_build/ --enable-shared --enable-zlib + make -j2 && make install + export HDF5_PLUGIN_PATH="${{ runner.workspace }}/vol-log-based-build/lib" + export HDF5_VOL_CONNECTOR="LOG under_vol=0;under_info={}" + make check + echo "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=${HDF5_VOL_CONNECTOR}" >> $GITHUB_ENV + + # Skip parallel testing for now as it appears to hang + - name: Test HDF5 Log-based VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Log-based VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . diff --git a/.github/workflows/vol_rest.yml b/.github/workflows/vol_rest.yml new file mode 100644 index 0000000..188e80d --- /dev/null +++ b/.github/workflows/vol_rest.yml @@ -0,0 +1,195 @@ +name: Test HDF5 REST VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_USERNAME: test_user1 + HSDS_PASSWORD: test + HSDS_PATH: /home/test_user1/ + HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ + HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock + HDF5_VOL_CONNECTOR: REST + ROOT_DIR: ${{github.workspace}}/hsdsdata + BUCKET_NAME: hsdstest + +jobs: + build_and_test: + name: Test HDF5 REST VOL connector + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install automake autoconf libtool libtool-bin libcurl4-openssl-dev libyajl-dev + + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + path: hdf5 + + - name: Configure HDF5 with REST VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_BUILD_HL_LIB:BOOL=ON \ + -DHDF5_TEST_API:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/HDFGroup/vol-rest.git" \ + -DHDF5_VOL_VOL-REST_BRANCH:STRING="master" \ + -DHDF5_VOL_VOL-REST_NAME:STRING="REST" \ + -DHDF5_VOL_VOL-REST_TEST_PARALLEL:BOOL=OFF \ + -DHDF5_VOL_REST_ENABLE_EXAMPLES=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 and REST VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout HSDS + uses: actions/checkout@v4 + with: + repository: HDFGroup/hsds + path: ${{github.workspace}}/hsds + + - name: Get HSDS HEAD commit SHA + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + export HSDS_COMMIT=`git rev-parse HEAD` + export HSDS_COMMIT_SHORT=`git rev-parse --short HEAD` + echo "HSDS_COMMIT=${HSDS_COMMIT}" >> $GITHUB_ENV + echo "HSDS_COMMIT_SHORT=${HSDS_COMMIT_SHORT}" >> $GITHUB_ENV + + # Note that we don't currently cache HSDS, as we would need + # to pick a fixed commit/tag in order to generate a reasonable + # key to use for caching/restoring from the cache + #- name: Restore HSDS (${{ env.HSDS_COMMIT_SHORT }}) installation cache + # id: restore-hsds + # uses: actions/cache@v3 + # with: + # path: ${{ runner.workspace }}/hsds-${{ env.HSDS_COMMIT_SHORT }}-install + # key: ${{ runner.os }}-${{ runner.arch }}-hsds-${{ env.HSDS_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Install HSDS (${{ env.HSDS_COMMIT_SHORT }}) dependencies + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + python -m pip install --upgrade pip + python -m pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install HSDS (${{ env.HSDS_COMMIT_SHORT }}) package + #if: ${{ ! steps.restore-hsds.outputs.cache-hit }} + shell: bash + run: | + cd ${{github.workspace}}/hsds + pip install -e . + + #- name: Cache HSDS (${{ env.HSDS_COMMIT_SHORT }}) installation + # uses: actions/cache/save@v3 + # if: ${{ ! steps.restore-hsds.outputs.cache-hit }} + # with: + # path: ${{ runner.workspace }}/hsds-${{ env.HSDS_COMMIT_SHORT }}-install + # key: ${{ runner.os }}-${{ runner.arch }}-hsds-${{ env.HSDS_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Run HSDS unit tests + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + pytest + + - name: Start HSDS + working-directory: ${{github.workspace}}/hsds + run: | + mkdir ${{github.workspace}}/hsdsdata && + mkdir ${{github.workspace}}/hsdsdata/hsdstest && + cp admin/config/groups.default admin/config/groups.txt && + cp admin/config/passwd.default admin/config/passwd.txt && + cp admin/config/groups.default admin/config/groups.txt && + cp admin/config/passwd.default admin/config/passwd.txt + ROOT_DIR=${{github.workspace}}/hsdsdata ./runall.sh --no-docker 1 & + sleep 10 + + - name: Test HSDS + working-directory: ${{github.workspace}}/hsds + run: | + python tests/integ/setup_test.py + + - name: Test HDF5 REST VOL connector with external tests + working-directory: ${{github.workspace}}/hdf5/build/ + run: | + sudo \ + HDF5_PLUGIN_PATH="${{ runner.workspace }}/hdf5_build/lib" \ + HDF5_VOL_CONNECTOR=REST \ + ADMIN_USERNAME=admin ADMIN_PASSWORD=admin \ + USER_NAME=test_user1 USER_PASSWORD=test \ + USER2_NAME=test_user2 USER2_PASSWORD=test \ + HSDS_USERNAME=test_user1 HSDS_PASSWORD=test \ + HSDS_PATH=/home/test_user1/ HDF5_API_TEST_PATH_PREFIX=/home/test_user1/ \ + HSDS_ENDPOINT=http+unix://%2Ftmp%2Fhs%2Fsn_1.sock \ + ROOT_DIR=${{github.workspace}}/hsdsdata \ + BUCKET_NAME=hsdstest \ + ctest --build-config ${{ inputs.build_mode }} -VV -R "test_rest_vol" . + + - name: Test HDF5 REST VOL connector with HDF5 API tests + working-directory: ${{github.workspace}}/hdf5/build/ + # Don't test the REST VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false + run: | + sudo \ + HDF5_PLUGIN_PATH="${{ runner.workspace }}/hdf5_build/lib" \ + HDF5_VOL_CONNECTOR=REST \ + ADMIN_USERNAME=admin ADMIN_PASSWORD=admin \ + USER_NAME=test_user1 USER_PASSWORD=test \ + USER2_NAME=test_user2 USER2_PASSWORD=test \ + HSDS_USERNAME=test_user1 HSDS_PASSWORD=test \ + HSDS_PATH=/home/test_user1/ HDF5_API_TEST_PATH_PREFIX=/home/test_user1/ \ + HSDS_ENDPOINT=http+unix://%2Ftmp%2Fhs%2Fsn_1.sock \ + ROOT_DIR=${{github.workspace}}/hsdsdata \ + BUCKET_NAME=hsdstest \ + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" . + + - name: Stop HSDS + working-directory: ${{github.workspace}}/hsds + run: | + ./stopall.sh |