diff options
author | Allen Byrne <50328838+byrnHDF@users.noreply.github.com> | 2024-01-03 17:23:42 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-03 17:23:42 (GMT) |
commit | 72e33ad7727765fd162a1f70ca502cc2437aabde (patch) | |
tree | 84b4cff56687fa7a8d443a529984932dd30a7fa4 /HDF5Examples/C/Perf | |
parent | 95827bc79d592ad5aa71ec3199a83ede9b324c20 (diff) | |
download | hdf5-72e33ad7727765fd162a1f70ca502cc2437aabde.zip hdf5-72e33ad7727765fd162a1f70ca502cc2437aabde.tar.gz hdf5-72e33ad7727765fd162a1f70ca502cc2437aabde.tar.bz2 |
Merge examples and workflows from develop (#3918)
Diffstat (limited to 'HDF5Examples/C/Perf')
-rw-r--r-- | HDF5Examples/C/Perf/CMakeLists.txt | 77 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/C_sourcefiles.cmake | 8 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/Makefile.am | 17 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/h5efc.c | 89 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/h5slab.h | 25 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/h5slabread.c | 49 | ||||
-rw-r--r-- | HDF5Examples/C/Perf/h5slabwrite.c | 141 |
7 files changed, 406 insertions, 0 deletions
diff --git a/HDF5Examples/C/Perf/CMakeLists.txt b/HDF5Examples/C/Perf/CMakeLists.txt new file mode 100644 index 0000000..e41def2 --- /dev/null +++ b/HDF5Examples/C/Perf/CMakeLists.txt @@ -0,0 +1,77 @@ +cmake_minimum_required (VERSION 3.12) +PROJECT (HDF5Examples_C_PERFORM) + +#----------------------------------------------------------------------------- +# Define Sources +#----------------------------------------------------------------------------- +include (C_sourcefiles.cmake) + +foreach (example ${examples}) + get_filename_component (example_name ${example} NAME_WE) + add_executable (${EXAMPLE_VARNAME}_${example_name} ${PROJECT_SOURCE_DIR}/${example}) + target_compile_options(${EXAMPLE_VARNAME}_${example_name} + PRIVATE + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_16_API}>:-DH5_USE_16_API>" + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_18_API}>:-DH5_USE_18_API>" + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_110_API}>:-DH5_USE_110_API>" + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_112_API}>:-DH5_USE_112_API>" + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_114_API}>:-DH5_USE_114_API>" + "$<$<BOOL:${${EXAMPLE_VARNAME}_USE_116_API}>:-DH5_USE_116_API>" + ) + if (H5_HAVE_PARALLEL) + target_include_directories (${EXAMPLE_VARNAME}_${example_name} PUBLIC ${MPI_C_INCLUDE_DIRS}) + endif () + target_link_libraries (${EXAMPLE_VARNAME}_${example_name} ${H5EX_HDF5_LINK_LIBS}) +endforeach () + +if (H5EX_BUILD_TESTING) + macro (ADD_H5_TEST testname) + add_test ( + NAME ${EXAMPLE_VARNAME}_${testname}-clearall + COMMAND ${CMAKE_COMMAND} + -E remove + ${testname}.h5 + ) + add_test ( + NAME ${EXAMPLE_VARNAME}_${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:${EXAMPLE_VARNAME}_${testname}>" + -D "TEST_ARGS:STRING=" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_REFERENCE=${testname}.tst" + -D "TEST_LIBRARY_DIRECTORY=${CMAKE_TEST_LIB_DIRECTORY}" + -P "${H5EX_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (${EXAMPLE_VARNAME}_${testname} PROPERTIES DEPENDS ${EXAMPLE_VARNAME}_${testname}-clearall) + if (HDF5_BUILD_TOOLS) + add_test ( + NAME ${EXAMPLE_VARNAME}_H5DUMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5EX_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=${ARGN};${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${testname}.dmp.out" + -D "TEST_EXPECT=0" + -D "TEST_REFERENCE=${testname}.dmp" + -D "TEST_LIBRARY_DIRECTORY=${CMAKE_TEST_LIB_DIRECTORY}" + -P "${H5EX_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (${EXAMPLE_VARNAME}_H5DUMP-${testname} PROPERTIES DEPENDS ${EXAMPLE_VARNAME}_${testname}) + endif () + endmacro () + + foreach (example ${examples}) + get_filename_component (example_name ${example} NAME_WE) + set (testdest "${PROJECT_BINARY_DIR}/${example_name}.test") + #message (STATUS " Copying ${example_name}.test") + add_custom_command ( + TARGET ${EXAMPLE_VARNAME}_${example_name} + POST_BUILD + COMMAND ${CMAKE_COMMAND} + ARGS -E copy_if_different ${PROJECT_SOURCE_DIR}/${example_name}.test ${testdest} + ) + ADD_H5_TEST (${example_name}) + endforeach () +endif () diff --git a/HDF5Examples/C/Perf/C_sourcefiles.cmake b/HDF5Examples/C/Perf/C_sourcefiles.cmake new file mode 100644 index 0000000..ac8a01d --- /dev/null +++ b/HDF5Examples/C/Perf/C_sourcefiles.cmake @@ -0,0 +1,8 @@ +#----------------------------------------------------------------------------- +# Define Sources, one file per application +#----------------------------------------------------------------------------- +set (examples + h5slabread.c + h5slabwrite.c + h5efc.c +) diff --git a/HDF5Examples/C/Perf/Makefile.am b/HDF5Examples/C/Perf/Makefile.am new file mode 100644 index 0000000..cac946e --- /dev/null +++ b/HDF5Examples/C/Perf/Makefile.am @@ -0,0 +1,17 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## + +noinst_PROGRAMS = h5slabread h5slabwrite h5efc + +EXTRA_DIST = + +AM_CPPFLAGS = '-D H5_USE_112_API' +TESTS = test.sh + +CLEANFILES = Makefile test.sh diff --git a/HDF5Examples/C/Perf/h5efc.c b/HDF5Examples/C/Perf/h5efc.c new file mode 100644 index 0000000..8e50c1d --- /dev/null +++ b/HDF5Examples/C/Perf/h5efc.c @@ -0,0 +1,89 @@ +/************************************************************ + + This example shows how to use the external file cache. + + This file is intended for use with HDF5 Library version + 1.8.7 or newer + + ************************************************************/ + +#include "hdf5.h" + +#define FILE "h5efc.h5" +#define EXT_FILE1 "h5efc1.h5" +#define EXT_FILE2 "h5efc2.h5" +#define EXT_FILE3 "h5efc3.h5" + +int +main(void) +{ + hid_t file1, file2, group, fapl; /* Handles */ + herr_t status; + + /* + * Create file access property list and set it to allow caching of open + * files visited through external links. + */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + status = H5Pset_elink_file_cache_size(fapl, 8); + + /* + * Create a new file using the file access property list. + */ + file1 = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + + /* + * Create files to serve as targets for external links. + */ + file2 = H5Fcreate(EXT_FILE1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + status = H5Fclose(file2); + file2 = H5Fcreate(EXT_FILE2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + status = H5Fclose(file2); + file2 = H5Fcreate(EXT_FILE3, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + status = H5Fclose(file2); + + /* + * Create external links to the target files. + */ + status = H5Lcreate_external(EXT_FILE1, "/", file1, "link_to_1", H5P_DEFAULT, H5P_DEFAULT); + status = H5Lcreate_external(EXT_FILE2, "/", file1, "link_to_2", H5P_DEFAULT, H5P_DEFAULT); + status = H5Lcreate_external(EXT_FILE3, "/", file1, "link_to_3", H5P_DEFAULT, H5P_DEFAULT); + + /* + * Open and close the targets of all three external links (these will be the + * root groups of the target files). The target files should be held open + * by the root file's external file cache after traversal. + */ + group = H5Gopen(file1, "/link_to_1", H5P_DEFAULT); + status = H5Gclose(group); + group = H5Gopen(file1, "/link_to_2", H5P_DEFAULT); + status = H5Gclose(group); + group = H5Gopen(file1, "/link_to_3", H5P_DEFAULT); + status = H5Gclose(group); + + /* + * Open and close the targets of all three external links again. The target + * files should already be held open by the root file's external file cache, + * so the library will not actually have to issue an "open" system call. + */ + group = H5Gopen(file1, "/link_to_1", H5P_DEFAULT); + status = H5Gclose(group); + group = H5Gopen(file1, "/link_to_2", H5P_DEFAULT); + status = H5Gclose(group); + group = H5Gopen(file1, "/link_to_3", H5P_DEFAULT); + status = H5Gclose(group); + + /* + * Release the root file's external file cache. This will close all the + * external link target files. + */ + status = H5Frelease_file_cache(file1); + + /* + * Close and release resources. + */ + status = H5Pclose(fapl); + status = H5Fclose(file1); + + return 0; +} diff --git a/HDF5Examples/C/Perf/h5slab.h b/HDF5Examples/C/Perf/h5slab.h new file mode 100644 index 0000000..c33feb7 --- /dev/null +++ b/HDF5Examples/C/Perf/h5slab.h @@ -0,0 +1,25 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Common definitions used by h5slab example programs (h5slabread, h5slabwrite). + * + * Created by Albert Cheng 2010/7/13. + */ +#include <stdlib.h> +#include <string.h> +#include "hdf5.h" +#if 1 +#define NX 65536 +#define NY 65536 /* dataset dimensions */ +#define CX 256 /* height of hyperslab */ +#define CY 4096 /* width of hyperslab */ +#else +#define NX 256 +#define NY 256 /* dataset dimensions */ +#define CX 32 /* height of hyperslab */ +#define CY 16 /* width of hyperslab */ +#endif +#define RC (NX / CX) +#define CC (NY / CY) diff --git a/HDF5Examples/C/Perf/h5slabread.c b/HDF5Examples/C/Perf/h5slabread.c new file mode 100644 index 0000000..7562b84 --- /dev/null +++ b/HDF5Examples/C/Perf/h5slabread.c @@ -0,0 +1,49 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include <string.h> + +#include "../Perf/h5slab.h" +#include "hdf5.h" + +/* Read the chunks in a row pattern. + * + * Created by Albert Cheng and Christian Chilan 2010/7/13. + */ +int +main(int argc, char **argv) +{ + hid_t file_id, dset_id, filespace, memspace, fapl, dxpl; + hsize_t dimsf[2], count[2], offset[2], chunk_dims[2] = {CX, CY}; + char *data, table[RC]; + unsigned long i, j, cx; + + fapl = H5Pcreate(H5P_FILE_ACCESS); + dxpl = H5Pcreate(H5P_DATASET_XFER); + fapl = dxpl = H5P_DEFAULT; + file_id = H5Fopen(argv[1], H5F_ACC_RDONLY, fapl); + dset_id = H5Dopen(file_id, "dataset1", H5P_DEFAULT); + filespace = H5Dget_space(dset_id); + count[0] = CX; + count[1] = NY; + memspace = H5Screate_simple(2, count, NULL); + + data = (char *)malloc(count[0] * count[1] * sizeof(char)); + for (i = 0; i < RC; i++) { + offset[0] = i * CX; + offset[1] = 0; + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + H5Dread(dset_id, H5T_NATIVE_CHAR, memspace, filespace, dxpl, data); + } + free(data); + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(dxpl); + H5Pclose(fapl); + H5Fclose(file_id); + return 0; +} diff --git a/HDF5Examples/C/Perf/h5slabwrite.c b/HDF5Examples/C/Perf/h5slabwrite.c new file mode 100644 index 0000000..c1af69e --- /dev/null +++ b/HDF5Examples/C/Perf/h5slabwrite.c @@ -0,0 +1,141 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* This example shows different data writing patterns to generate data files + * that will exhibit substantially different data read speed. The files + * contain 2-D chunk storage datasets. + * The 2 writing patterns are: + * 1. Random--chunks are wriitten in the random order. + * 2. ByRow--chunks are written by row order. + */ + +#include <stdlib.h> +#include <string.h> + +#include "../Perf/h5slab.h" +#include "hdf5.h" + +/* Write the chunks in the row order. This provides good and bad read + * performance if the read pattern is by row and by column respectively. + * + * Created by Albert Cheng and Christian Chilan 2010/7/13. + */ +int +createfilebyrow(void) +{ + hid_t file_id, dset_id, filespace, memspace, fapl, dxpl, dcpl; + hsize_t dimsf[2], count[2], offset[2], chunk_dims[2] = {CX, CY}; + char *data, dataval, table[RC]; + unsigned long i, j, l, cx; + fapl = H5Pcreate(H5P_FILE_ACCESS); + dcpl = H5Pcreate(H5P_DATASET_CREATE); + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_chunk(dcpl, 2, chunk_dims); + fapl = dxpl = H5P_DEFAULT; + file_id = H5Fcreate("row_alloc.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + dimsf[0] = NX; + dimsf[1] = NY; + filespace = H5Screate_simple(2, dimsf, NULL); + dset_id = H5Dcreate(file_id, "dataset1", H5T_NATIVE_CHAR, filespace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + count[0] = CX; + count[1] = NY; + memspace = H5Screate_simple(2, count, NULL); + + data = (char *)malloc(count[0] * count[1] * sizeof(char)); + + /* writing the whole chunked rows each time. */ + for (l = 0; l < RC; l++) { + + offset[0] = l * CX; + offset[1] = 0; + + /* fill with values according to row number */ + for (i = 0; i < count[0]; i++) + for (j = 0; j < count[1]; j++) + data[i * count[1] + j] = l; + + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + H5Dwrite(dset_id, H5T_NATIVE_CHAR, memspace, filespace, dxpl, data); + } + + free(data); + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(dxpl); + H5Pclose(dcpl); + H5Pclose(fapl); + H5Fclose(file_id); + return 0; +} + +/* Write the chunks in a random pattern. This provides a read performance + * worse than when the chunks are written and read in the same order, whether + * it is by row or by column. + * + * Created by Albert Cheng and Christian Chilan 2010/7/13. + */ +int +createfilerandom(void) +{ + hid_t file_id, dset_id, filespace, memspace, fapl, dxpl, dcpl; + hsize_t dimsf[2], count[2], offset[2], chunk_dims[2] = {CX, CY}; + char *data, table[RC][CC]; + unsigned long i, j, cx, cy; + fapl = H5Pcreate(H5P_FILE_ACCESS); + dcpl = H5Pcreate(H5P_DATASET_CREATE); + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_chunk(dcpl, 2, chunk_dims); + fapl = dxpl = H5P_DEFAULT; + file_id = H5Fcreate("random_alloc.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + dimsf[0] = NX; + dimsf[1] = NY; + filespace = H5Screate_simple(2, dimsf, NULL); + dset_id = H5Dcreate(file_id, "dataset1", H5T_NATIVE_CHAR, filespace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + count[0] = CX; + count[1] = CY; + memspace = H5Screate_simple(2, count, NULL); + data = (char *)malloc(count[0] * count[1] * sizeof(char)); + + for (i = 0; i < RC; i++) + for (j = 0; j < CC; j++) + table[i][j] = 0; + + for (i = 0; i < RC * CC; i++) { + do { + cx = rand() % RC; + cy = rand() % CC; + } while (table[cx][cy]); + + for (j = 0; j < count[0] * count[1]; j++) { + data[j] = cx + cy; + } + + table[cx][cy] = 1; + + offset[0] = cx * CX; + offset[1] = cy * CY; + + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + H5Dwrite(dset_id, H5T_NATIVE_CHAR, memspace, filespace, dxpl, data); + } + + free(data); + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(dxpl); + H5Pclose(dcpl); + H5Pclose(fapl); + H5Fclose(file_id); + return 0; +} + +int +main(int argc, char **argv) +{ + createfilebyrow(); + createfilerandom(); +} |