diff options
author | Scot Breitenfeld <brtnfld@hdfgroup.org> | 2023-11-07 15:50:03 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-11-07 15:50:03 (GMT) |
commit | ec695cf34f7256d7af9bbc2fcf5366da0119fcea (patch) | |
tree | fa723a5125fd0981177c8207cf582a39f2ed1430 /utils/subfiling_vfd/h5fuse.in | |
parent | 3c0714797196fe428e1b81f3c7b02149676c54f7 (diff) | |
download | hdf5-ec695cf34f7256d7af9bbc2fcf5366da0119fcea.zip hdf5-ec695cf34f7256d7af9bbc2fcf5366da0119fcea.tar.gz hdf5-ec695cf34f7256d7af9bbc2fcf5366da0119fcea.tar.bz2 |
Renamed h5fuse.sh to h5fuse (#3834)
* provide an alternative to mapfile for older bash
Diffstat (limited to 'utils/subfiling_vfd/h5fuse.in')
-rwxr-xr-x | utils/subfiling_vfd/h5fuse.in | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/utils/subfiling_vfd/h5fuse.in b/utils/subfiling_vfd/h5fuse.in new file mode 100755 index 0000000..a4c6a05 --- /dev/null +++ b/utils/subfiling_vfd/h5fuse.in @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +BLD='\033[1m' +GRN='\033[0;32m' +RED='\033[0;31m' +CYN='\033[0;36m' +NC='\033[0m' # No Color + +############################################################ +# Usage # +############################################################ +function usage { + echo "" + # Display usage + echo "Purpose: Combine subfiles into a single HDF5 file. Requires the subfiling + configuration file either as a command-line argument or the script will + search for the *.config file in the current directory." + echo "" + echo "usage: h5fuse [-f filename] [-h] [-p] [-q] [-r] [-v] " + echo "-f filename Subfile configuration file." + echo "-h Print this help." + echo "-q Quiet all output. [no]" + echo "-p h5fuse is being run in parallel, with more than one rank. [no]" + echo "-r Remove subfiles after being processed. [no]" + echo "-v Verbose output. [no]" + echo "" +} + +function gen_mpi { + +# Program to determine MPI rank and size if being run in parallel (-p). + +cat > "${c_src}" << EOL +#include <mpi.h> +#include <stdio.h> +int main() { + MPI_Init(NULL, NULL); + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + printf("%d %d", world_rank, world_size); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); +} +EOL + +} + +function fuse { + +# function for fusing the files + +mpi_rank=0 +mpi_size=1 +nstart=1 +nend=$nsubfiles + +if [ "$parallel" == "true" ]; then + + hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom) + c_exec="h5fuse_"${hex} + c_src=${c_exec}.c + + # Generate and compile an MPI program to get MPI rank and size + if [ ! -f "${c_src}" ]; then + gen_mpi + CC=@CC@ + ${CC} "${c_src}" -o "${c_exec}" + fi + wait + rank_size=$(./"${c_exec}") + read -r mpi_rank mpi_size <<<"$rank_size" + + rm -f "${c_src}" "${c_exec}" + + # Divide the subfiles among the ranks + iwork1=$(( nsubfiles / mpi_size )) + iwork2=$(( nsubfiles % mpi_size )) + min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 )) + nstart=$(( mpi_rank * iwork1 + 1 + min )) + nend=$(( nstart + iwork1 - 1 )) + if [ $iwork2 -gt "$mpi_rank" ]; then + nend=$(( nend + 1 )) + fi +fi + +############################################################ +# COMBINE SUBFILES INTO AN HDF5 FILE # +############################################################ +icnt=1 +skip=0 +seek=0 +seek_cnt=0 +for i in "${subfiles[@]}"; do + + subfile="${subfile_dir}/${i}" + + # bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs + # ibs=BYTES read up to BYTES bytes at a time + # obs=BYTES write BYTES bytes at a time + # seek=N skip N obs-sized blocks at start of output + # skip=N skip N ibs-sized blocks at start of input + + status=1 + fsize=${subfiles_size[icnt-1]} + if [ "$fsize" -eq "0" ]; then + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + if [ "$rm_subf" == "true" ]; then + if [ -f "${subfile}" ]; then + \rm -f "$subfile" + fi + fi + else + if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then + records_left=$fsize + while [ "$status" -gt 0 ]; do + if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then + EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc" + if [ "$verbose" == "true" ]; then + echo -e "$GRN $EXEC $NC" + fi + err=$( $EXEC 2>&1 1>/dev/null ) + if [ $? -ne 0 ]; then + echo -e "$CYN ERR: dd Utility Failed $NC" + echo -e "$CYN MSG: $err $NC" + exit $FAILED + fi + records_left=$((records_left-stripe_size)) + skip=$((skip+1)) + seek=$((seek_cnt+skip*nsubfiles)) + else + status=0 + skip=0 + fi + done; wait + if [ "$rm_subf" == "true" ]; then + \rm -f "$subfile" + fi + fi + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + fi + icnt=$(( icnt +1 )) +done; wait + +} + +############################################################ +############################################################ +# Main program # +############################################################ +############################################################ + +############################################################ +# Process the input options. Add options as needed. # +############################################################ +# Get the options +file_config="" +verbose="false" +quiet="false" +rm_subf="false" +parallel="false" +while getopts "hpqrvf:" option; do + case $option in + f) # subfiling configuration file + file_config=$OPTARG;; + h) # display Help + usage + exit;; + p) # HDF5 fused file + parallel="true";; + q) # quiet all output + quiet="true";; + r) # remove completed subfiles + rm_subf="true";; + v) # verbose output + verbose="true";; + \?) # Invalid option + echo -e "$RED ERROR: Invalid option ${BLD}-${OPTARG}${RED} $NC" + usage + exit 1;; + * ) usage + exit 1;; + esac +done + +FAILED=1 +############################################################ +# Configure file checks # +############################################################ +# +SUBF_CONFDIR="${H5FD_SUBFILING_CONFIG_FILE_PREFIX:-$PWD}" + +# Try to find the config file +if [ -z "$file_config" ]; then + nfiles=$(find "$SUBF_CONFDIR" -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m) + if [[ "$nfiles" != "1" ]]; then + if [[ "$nfiles" == "0" ]]; then + echo -e "$RED Failed to find .config file in ${SUBF_CONFDIR} $NC" + usage + exit $FAILED + else + echo -e "$RED More than one .config file found in ${SUBF_CONFDIR} $NC" + usage + exit $FAILED + fi + fi + file_config=$(find "${SUBF_CONFDIR}" -maxdepth 1 -type f -iname '*.config') +fi + +if [ ! -f "$file_config" ]; then + echo -e "${RED} configuration file ${BLD}$file_config${NC} ${RED}does not exist. $NC" + exit $FAILED +fi + +stripe_size=$(grep "stripe_size=" "$file_config" | cut -d "=" -f2) +if test -z "$stripe_size"; then + echo -e "$RED failed to find stripe_size in $file_config $NC" + exit $FAILED +fi + +hdf5_file="$(grep "hdf5_file=" "$file_config" | cut -d "=" -f2)" +if test -z "$hdf5_file"; then + echo -e "$RED failed to find hdf5 output file in $file_config $NC" + exit $FAILED +fi + +subfile_dir="$(grep "subfile_dir=" "$file_config" | cut -d "=" -f2)" +if test -z "$subfile_dir"; then + echo -e "$RED failed to find subfile directory in $file_config $NC" + exit $FAILED +fi + +subfs=$(sed -e '1,/subfile_dir=/d' "$file_config") +if command -v mapfile > /dev/null; then + # For bash 4.4+ + mapfile -t subfiles <<< "$subfs" +else + while IFS= read -r line; do + subfiles+=("$line") + done <<< "$subfs" +fi +if [ ${#subfiles[@]} -eq 0 ]; then + echo -e "$RED failed to find subfiles list in $file_config $NC" + exit $FAILED +fi +nsubfiles=${#subfiles[@]} + +# Get the number of local subfiles +subfiles_loc=() +subfiles_size=() +for i in "${subfiles[@]}"; do + subfile="${subfile_dir}/${i}" + if [ -f "${subfile}" ]; then + subfiles_loc+=("$subfile") + subfiles_size+=($(wc -c "${subfile}" | awk '{print $1}')) + else + subfiles_size+=(0) + fi +done + +if [ "$quiet" == "false" ]; then + TIMEFORMAT="COMPLETION TIME = %R s" + time fuse +else + fuse +fi |