diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/test/perform/direct_write_perf.c | 684 |
1 files changed, 684 insertions, 0 deletions
diff --git a/tools/test/perform/direct_write_perf.c b/tools/test/perform/direct_write_perf.c new file mode 100644 index 0000000..f13cd24 --- /dev/null +++ b/tools/test/perform/direct_write_perf.c @@ -0,0 +1,684 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * This tests the performance of the H5Dwrite_chunk() function. + * + */ + +#include "hdf5.h" + +#ifdef H5_HAVE_FILTER_DEFLATE +#include <zlib.h> + +#if !defined(WIN32) && !defined(__MINGW32__) + +#include <math.h> + +#ifdef H5_STDC_HEADERS +# include <errno.h> +# include <fcntl.h> +# include <stdio.h> +# include <stdlib.h> +#endif + +#ifdef H5_HAVE_UNISTD_H +# include <sys/types.h> +# include <unistd.h> +#endif + +#ifdef H5_HAVE_SYS_STAT_H +# include <sys/stat.h> +#endif + +#if defined(H5_TIME_WITH_SYS_TIME) +# include <sys/time.h> +# include <time.h> +#elif defined(H5_HAVE_SYS_TIME_H) +# include <sys/time.h> +#else +# include <time.h> +#endif + +const char *FILENAME[] = { + "direct_write", + "unix.raw", + NULL +}; + +/* + * Print the current location on the standard output stream. + */ +#define FUNC __func__ +#define AT() printf (" at %s:%d in %s()...\n", \ + __FILE__, __LINE__, FUNC); +#define H5_FAILED() {puts("*FAILED*");fflush(stdout);} +#define TEST_ERROR {H5_FAILED(); AT(); goto error;} +#define TESTING(WHAT) {printf("Testing %-62s",WHAT); fflush(stdout);} +#define PASSED() {puts(" PASSED");fflush(stdout);} + +#define DIRECT_UNCOMPRESSED_DSET "direct_uncompressed_dset" +#define DIRECT_COMPRESSED_DSET "direct_compressed_dset" +#define REG_COMPRESSED_DSET "reg_compressed_dset" +#define REG_NO_COMPRESS_DSET "reg_no_compress_dset" +#define RANK 3 +#define NX 100 +#define NY 1000 +#define NZ 250 +#define CHUNK_NX 1 +#define CHUNK_NY 1000 +#define CHUNK_NZ 250 + +#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) +char filename[1024]; +unsigned int *outbuf[NX]; +size_t data_size[NX]; +double total_size = 0.0; +unsigned int *direct_buf[NX]; +double MB = 1048576.0; + +/*-------------------------------------------------- + * Function to report IO rate + *-------------------------------------------------- + */ +void reportTime(struct timeval start, double mbytes) +{ + struct timeval timeval_stop,timeval_diff; + + /*end timing*/ + gettimeofday(&timeval_stop,NULL); + + /* Calculate the elapsed gettimeofday time */ + timeval_diff.tv_usec=timeval_stop.tv_usec-start.tv_usec; + timeval_diff.tv_sec=timeval_stop.tv_sec-start.tv_sec; + + if(timeval_diff.tv_usec<0) { + timeval_diff.tv_usec+=1000000; + timeval_diff.tv_sec--; + } /* end if */ + +/*printf("mbytes=%lf, sec=%lf, usec=%lf\n", mbytes, (double)timeval_diff.tv_sec, (double)timeval_diff.tv_usec);*/ + printf("MBytes/second: %lf\n", (double)mbytes/((double)timeval_diff.tv_sec+((double)timeval_diff.tv_usec/(double)1000000.0))); +} + +/*-------------------------------------------------- + * Create file, datasets, and initialize data + *-------------------------------------------------- + */ +int create_file(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t fapl; + hid_t cparms; + hid_t dataspace, dataset; + hsize_t dims[RANK] = {NX, NY, NZ}; + hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; + unsigned int aggression = 9; /* Compression aggression setting */ + int ret; + int i, j, n; + + int flag; + int unix_file; + + unsigned int *p; + size_t buf_size = CHUNK_NY*CHUNK_NZ*sizeof(unsigned int); + + const Bytef *z_src; + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + + TESTING("Create a file and dataset"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* + * Create a new file. If file exists its contents will be overwritten. + */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_chunk( cparms, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DIRECT_UNCOMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + if((dataset = H5Dcreate2(file, REG_NO_COMPRESS_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + /* Set compression */ + if(H5Pset_deflate( cparms, aggression) < 0) + TEST_ERROR; + + if((dataset = H5Dcreate2(file, DIRECT_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + + if((dataset = H5Dcreate2(file, REG_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + if(H5Fclose(file) < 0) + TEST_ERROR; + + if(H5Sclose(dataspace) < 0) + TEST_ERROR; + + if(H5Pclose(cparms) < 0) + TEST_ERROR; + + /* create a unix file*/ + flag = O_CREAT|O_TRUNC|O_WRONLY; + + if ((unix_file=open(FILENAME[1],flag,S_IRWXU))== -1) + TEST_ERROR; + + if (close(unix_file) < 0) + { + printf(" unable to close the file\n"); + TEST_ERROR; + } + + + /* Initialize data for chunks */ + for(i = 0; i < NX; i++) { + p = direct_buf[i] = (unsigned int*)malloc(CHUNK_NY*CHUNK_NZ*sizeof(unsigned int)); + + for(j=0; j < CHUNK_NY*CHUNK_NZ; j++, p++) + *p = rand() % 65000; + + z_src = (const Bytef*)direct_buf[i]; + + z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + /* Allocate output (compressed) buffer */ + outbuf[i] = (unsigned int*)malloc((size_t)z_dst_nbytes); + z_dst = (Bytef *)outbuf[i]; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + data_size[i] = (size_t)z_dst_nbytes; + total_size += data_size[i]; + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + fprintf(stderr, "overflow"); + TEST_ERROR; + } else if(Z_MEM_ERROR == ret) { + fprintf(stderr, "deflate memory error"); + TEST_ERROR; + } else if(Z_OK != ret) { + fprintf(stderr, "other deflate error"); + TEST_ERROR; + } + } + + + PASSED(); + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance of the new function + * with precompressed data. + *-------------------------------------------------- + */ +int +test_direct_write_uncompressed_data(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t dxpl; + herr_t status; + int i; + + unsigned filter_mask = 0; + hsize_t offset[RANK] = {0, 0, 0}; + + struct timeval timeval_start; + + TESTING("H5Dwrite_chunk for uncompressed data"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen2(file, DIRECT_UNCOMPRESSED_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i<NX; i++) { + status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, CHUNK_NY*CHUNK_NZ*sizeof(unsigned int), direct_buf[i]); + (offset[0])++; + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Pclose(dxpl); + H5Fclose(file); + + /* Report the performance */ + reportTime(timeval_start, (double)(NX*NY*NZ*sizeof(unsigned int)/MB)); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Pclose(dxpl); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + + +/*-------------------------------------------------- + * Benchmark the performance of the new function + * with precompressed data. + *-------------------------------------------------- + */ +int +test_direct_write_compressed_data(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t dxpl; + herr_t status; + int i; + + unsigned filter_mask = 0; + hsize_t offset[RANK] = {0, 0, 0}; + + struct timeval timeval_start; + + TESTING("H5DOwrite_chunk for pre-compressed data"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen2(file, DIRECT_COMPRESSED_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i<NX; i++) { + status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, data_size[i], outbuf[i]); + (offset[0])++; + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Pclose(dxpl); + H5Fclose(file); + + /* Report the performance */ + reportTime(timeval_start, (double)(total_size/MB)); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Pclose(dxpl); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance of the regular H5Dwrite + * with compression filter enabled. + *-------------------------------------------------- + */ +int +test_compressed_write(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t mem_space; + hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; + hid_t dxpl; + herr_t status; + int i; + + hsize_t start[RANK]; /* Start of hyperslab */ + hsize_t stride[RANK]; /* Stride of hyperslab */ + hsize_t count[RANK]; /* Block count */ + hsize_t block[RANK]; /* Block sizes */ + + struct timeval timeval_start; + + TESTING("H5Dwrite with compression enabled"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen2(file, REG_COMPRESSED_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if((dataspace = H5Dget_space(dataset)) < 0) + TEST_ERROR; + + start[0] = start[1] = start[2] = 0; + stride[0] = stride[1] = stride[2] = 1; + count[0] = count[1] = count[2] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; block[2] = CHUNK_NZ; + + for(i=0; i<NX; i++) { + /* + * Select hyperslab for one chunk in the file + */ + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + TEST_ERROR; + (start[0])++; + + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, mem_space, dataspace, + H5P_DEFAULT, direct_buf[i])) < 0) + TEST_ERROR; + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(mem_space); + H5Pclose(dxpl); + H5Fclose(file); + + /* Report the performance */ + reportTime(timeval_start, (double)(NX*NY*NZ*sizeof(unsigned int)/MB)); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(mem_space); + H5Pclose(dxpl); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance of the regular H5Dwrite + * with compression + *-------------------------------------------------- + */ +int +test_no_compress_write(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t mem_space; + hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; + hid_t dxpl; + herr_t status; + int i; + + hsize_t start[RANK]; /* Start of hyperslab */ + hsize_t stride[RANK]; /* Stride of hyperslab */ + hsize_t count[RANK]; /* Block count */ + hsize_t block[RANK]; /* Block sizes */ + + struct timeval timeval_start; + + TESTING("H5Dwrite without compression"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen2(file, REG_NO_COMPRESS_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if((dataspace = H5Dget_space(dataset)) < 0) + TEST_ERROR; + + start[0] = start[1] = start[2] = 0; + stride[0] = stride[1] = stride[2] = 1; + count[0] = count[1] = count[2] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; block[2] = CHUNK_NZ; + + for(i=0; i<NX; i++) { + /* + * Select hyperslab for one chunk in the file + */ + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + TEST_ERROR; + (start[0])++; + + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, mem_space, dataspace, + H5P_DEFAULT, direct_buf[i])) < 0) + TEST_ERROR; + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(mem_space); + H5Pclose(dxpl); + H5Fclose(file); + + /* Report the performance */ + reportTime(timeval_start, (double)(NX*NY*NZ*sizeof(unsigned int)/MB)); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(mem_space); + H5Pclose(dxpl); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance for writing compressed + * data to a Unix file + *-------------------------------------------------- + */ +int +test_unix_write(void) +{ + int file, flag; + ssize_t op_size; + int i; + struct timeval timeval_start; + + TESTING("Write compressed data to a Unix file"); + + /* create file*/ + flag = O_WRONLY; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + if ((file=open(FILENAME[1],flag))== -1) + TEST_ERROR; + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i<NX; i++) { + op_size = write(file, outbuf[i],data_size[i]); + if (op_size < 0) + { + printf(" Error in writing data to file because %s \n", strerror(errno)); + TEST_ERROR; + } + else if (op_size == 0) + { + printf(" unable to write sufficent data to file because %s \n", strerror(errno)); + TEST_ERROR; + } + } + + if (close(file) < 0) + { + printf(" unable to close the file\n"); + TEST_ERROR; + } + + /* Report the performance */ + reportTime(timeval_start, (double)(total_size/MB)); + + PASSED(); + return 0; + +error: + return 1; +} + +/*-------------------------------------------------- + * Main function + *-------------------------------------------------- + */ +int +main (void) +{ + hid_t fapl = H5P_DEFAULT; + int i; + + sprintf(filename, "%s.h5", FILENAME[0]); + + create_file(fapl); + test_direct_write_uncompressed_data(fapl); + test_direct_write_compressed_data(fapl); + test_no_compress_write(fapl); + test_compressed_write(fapl); + test_unix_write(); + + for(i=0; i<NX; i++) { + free(outbuf[i]); + free(direct_buf[i]); + } + + return 0; +} + +#else /* WIN32 / MINGW32 */ + +int +main(void) +{ + printf("Non-POSIX platform. Exiting.\n"); + return EXIT_FAILURE; +} /* end main() */ + +#endif /* WIN32 / MINGW32 */ + +#else /* !H5_HAVE_FILTER_DEFLATE */ + +/* + * Function: main + * Purpose: Dummy main() function for if HDF5 was configured without + * zlib stuff. + * Return: EXIT_SUCCESS + */ +int +main(void) +{ + HDfprintf(stdout, "No compression IO performance because zlib was not configured\n"); + return EXIT_SUCCESS; +} + +#endif /* !H5_HAVE_FILTER_DEFLATE */ + |