diff options
Diffstat (limited to 'utils')
| -rwxr-xr-x | utils/subfiling_vfd/h5fuse.sh.in | 222 |
1 files changed, 163 insertions, 59 deletions
diff --git a/utils/subfiling_vfd/h5fuse.sh.in b/utils/subfiling_vfd/h5fuse.sh.in index 48e3e61..2085033 100755 --- a/utils/subfiling_vfd/h5fuse.sh.in +++ b/utils/subfiling_vfd/h5fuse.sh.in @@ -14,6 +14,7 @@ BLD='\033[1m' GRN='\033[0;32m' RED='\033[0;31m' PUR='\033[0;35m' +CYN='\033[0;36m' NC='\033[0m' # No Color ############################################################ @@ -23,15 +24,40 @@ function usage() { echo "" # Display usage echo "Purpose: Combine subfiles into a single HDF5 file. Requires the subfiling - configuration file either as a command-line argument, or the script will + configuration file either as a command-line argument or the script will search for the *.config file in the current directory." echo "" - echo "usage: h5fuse.sh [-h] [-f filename]" - echo "-h Print this help." - echo "-f filename Subfile configuration file." + echo "usage: h5fuse.sh [-f filename] [-h] [-p] [-q] [-r] [-v] " + echo "-f filename Subfile configuration file." + echo "-h Print this help." + echo "-q Quiet all output. [no]" + echo "-p h5fuse.sh is being run in parallel, with more than one rank. [no]" + echo "-r Remove subfiles after being processed. [no]" + echo "-v Verbose output. [no]" echo "" } +function gen_mpi() { + +# Program to determine MPI rank and size if being run in parallel (-p). + +cat > "${c_src}" << EOL +#include <mpi.h> +#include <stdio.h> +int main() { + MPI_Init(NULL, NULL); + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + printf("%d %d", world_rank, world_size); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); +} +EOL + +} + ############################################################ ############################################################ # Main program # @@ -43,14 +69,25 @@ function usage() { ############################################################ # Get the options file_config="" - -while getopts ":h:f:" option; do +verbose="false" +quiet="false" +rm_subf="false" +parallel="false" +while getopts "hpqrvf:" option; do case $option in + f) # subfiling configuration file + file_config=$OPTARG;; h) # display Help usage exit;; - f) # subfiling configuration file - file_config=$OPTARG;; + p) # HDF5 fused file + parallel="true";; + q) # quiet all output + quiet="true";; + r) # remove completed subfiles + rm_subf="true";; + v) # verbose output + verbose="true";; \?) # Invalid option echo -e "$RED ERROR: Invalid option ${BLD}-${OPTARG}${RED} $NC" usage @@ -61,24 +98,27 @@ while getopts ":h:f:" option; do done FAILED=1 -nfiles=1 ############################################################ # Configure file checks # ############################################################ +# +SUBF_CONFDIR="${H5FD_SUBFILING_CONFIG_FILE_PREFIX:-$PWD}" + +# Try to find the config file if [ -z "$file_config" ]; then - nfiles=$(find . -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m) + nfiles=$(find "$SUBF_CONFDIR" -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m) if [[ "$nfiles" != "1" ]]; then if [[ "$nfiles" == "0" ]]; then - echo -e "$RED Failed to find .config file in current directory. $NC" + echo -e "$RED Failed to find .config file in ${SUBF_CONFDIR} $NC" usage exit $FAILED else - echo -e "$RED More than one .config file found in current directory. $NC" + echo -e "$RED More than one .config file found in ${SUBF_CONFDIR} $NC" usage exit $FAILED fi fi - file_config=$(find . -maxdepth 1 -type f -iname "*.config") + file_config=$(find "${SUBF_CONFDIR}" -maxdepth 1 -type f -iname '*.config') fi if [ ! -f "$file_config" ]; then @@ -104,62 +144,126 @@ if test -z "$subfile_dir"; then exit $FAILED fi -subfiles=( $( sed -e '1,/subfile_dir=/d' "$file_config" ) ) -#for i in "${subfiles[@]}"; do -# echo "$i" -#done +# For bash 4.4+ +subfs=$(sed -e '1,/subfile_dir=/d' "$file_config") +mapfile -t subfiles <<< "$subfs" if [ ${#subfiles[@]} -eq 0 ]; then echo -e "$RED failed to find subfiles list in $file_config $NC" exit $FAILED fi +nsubfiles=${#subfiles[@]} + +# Get the number of local subfiles +subfiles_loc=() +subfiles_size=() +for i in "${subfiles[@]}"; do + subfile="${subfile_dir}/${i}" + if [ -f "${subfile}" ]; then + subfiles_loc+=("$subfile") + subfiles_size+=($(wc -c "${subfile}" | awk '{print $1}')) + else + subfiles_size+=(0) + fi +done + +START="$(date +%s%N)" -rm -f "$hdf5_file" +mpi_rank=0 +mpi_size=1 +nstart=1 +nend=$nsubfiles -## COMBINE SUBFILES INTO AN HDF5 FILE ## +if [ "$parallel" == "true" ]; then + hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom) + c_exec="h5fuse_"${hex} + c_src=${c_exec}.c + + # Generate and compile an MPI program to get MPI rank and size + if [ ! -f "${c_src}" ]; then + gen_mpi + CC=@CC@ + ${CC} "${c_src}" -o "${c_exec}" + fi + wait + rank_size=$(./"${c_exec}") + read -r mpi_rank mpi_size <<<"$rank_size" + + rm -f "${c_src}" "${c_exec}" + + # Divide the subfiles among the ranks + iwork1=$(( nsubfiles / mpi_size )) + iwork2=$(( nsubfiles % mpi_size )) + min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 )) + nstart=$(( mpi_rank * iwork1 + 1 + min )) + nend=$(( nstart + iwork1 - 1 )) + if [ $iwork2 -gt "$mpi_rank" ]; then + nend=$(( nend + 1 )) + fi +fi + +############################################################ +# COMBINE SUBFILES INTO AN HDF5 FILE # +############################################################ +icnt=1 skip=0 -status=$nfiles -START="$(date +%s%N)" -while [ "$status" -gt 0 ]; do - icnt=0 - for i in "${subfiles[@]}"; do - subfile="${subfile_dir}/${i}" - # Verify the file exists - if [ ! -f "${subfile}" ]; then - echo -e "$RED ERROR: file \"${subfile}\" does not exist. $NC" - exit $FAILED - fi +seek=0 +seek_cnt=0 +for i in "${subfiles[@]}"; do - # Verify the file is not being accessed by a process - t_max=60 - t_sleep=1 - t_elapsed=0 + subfile="${subfile_dir}/${i}" - while fuser -s "${subfile}"; do - if [[ $((t_elapsed % 5)) -eq 0 ]]; then - echo -e "$GRN waiting for process to finish accessing file \"${subfile}\" ... [${t_elapsed}s/${t_max}s] $NC" - fi - sleep $t_sleep - t_elapsed=$((t_elapsed+t_sleep)) - if [[ $t_elapsed -ge $t_max ]]; then - echo -e "$RED ERROR: file \"${subfile}\" still has process accessing it after ${t_elapsed}s $NC" - exit $FAILED + # bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs + # ibs=BYTES read up to BYTES bytes at a time + # obs=BYTES write BYTES bytes at a time + # seek=N skip N obs-sized blocks at start of output + # skip=N skip N ibs-sized blocks at start of input + + status=1 + fsize=${subfiles_size[icnt-1]} + if [ "$fsize" -eq "0" ]; then + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + if [ "$rm_subf" == "true" ]; then + if [ -f "${subfile}" ]; then + \rm -f "$subfile" + fi + fi + else + if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then + records_left=$fsize + while [ "$status" -gt 0 ]; do + if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then + EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc" + if [ "$verbose" == "true" ]; then + echo -e "$GRN $EXEC $NC" + fi + err=$( $EXEC 2>&1 1>/dev/null ) + if [ $? -ne 0 ]; then + echo -e "$CYN ERR: dd Utility Failed $NC" + echo -e "$CYN MSG: $err $NC" + exit $FAILED + fi + records_left=$((records_left-stripe_size)) + skip=$((skip+1)) + seek=$((seek_cnt+skip*nsubfiles)) + else + status=0 + skip=0 + fi + done; wait + if [ "$rm_subf" == "true" ]; then + \rm -f "$subfile" fi - done - - fsize=$(wc -c "${subfile}" | awk '{print $1}') - if [ $((skip*stripe_size)) -le "$fsize" ]; then - EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip oflag=append conv=notrunc" - echo -e "$GRN $EXEC $NC" - err="$( $EXEC 2>&1 > /dev/null &)" - icnt=$((icnt+1)) - else - subfiles=("${subfiles[@]:0:icnt}" "${subfiles[@]:$((icnt+1))}") - status=${#subfiles[@]} - fi - done; wait - skip=$((skip+1)) -done + fi + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + fi + icnt=$(( icnt +1 )) +done; wait + END=$(( $(date +%s%N) - START )) DURATION_SEC=$(awk -vp="$END" -vq=0.000000001 'BEGIN{printf "%.4f" ,p * q}') -echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC" +if [ "$quiet" == "false" ]; then + echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC" +fi
\ No newline at end of file |