path: root/tools/test/perform/chunk_cache.c

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Board of Trustees of the University of Illinois.         *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the root of the source code       *
 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
 * If you do not have access to either file, you may request a copy from     *
 * help@hdfgroup.org.                                                        *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/*
 *  Purpose: check the performance of chunk cache in these two cases (HDFFV-10601):
 *          1. partial chunks exist along any dimension.
 *          2. number of slots in chunk cache is smaller than the number of chunks
 *             in the fastest-growing dimension.
 */
#include "hdf5.h"
#include "H5private.h"

#define FILENAME    "chunk_cache_perf.h5"

#define RANK        2

#define DSET1_NAME  "partial_chunks"
#define DSET1_DIM1   9 * 1000
#define DSET1_DIM2   9
#define CHUNK1_DIM1  2 * 1000
#define CHUNK1_DIM2  2

#define DSET2_NAME  "hash_value"
#define DSET2_DIM1   300
#define DSET2_DIM2   600
#define CHUNK2_DIM1  100
#define CHUNK2_DIM2  100

#define RDCC_NSLOTS  5
#define RDCC_NBYTES  1024 * 1024 * 10
#define RDCC_W0      0.75F

#define FILTER_COUNTER    306
static size_t    nbytes_global;

typedef struct test_time_t {
    long tv_sec;
    long tv_usec;
} test_time_t;

/* Local function prototypes for the dummy filter */
static size_t
counter (unsigned flags, size_t cd_nelmts,
         const unsigned *cd_values, size_t nbytes,
         size_t *buf_size, void **buf);

/* This message derives from H5Z */
const H5Z_class2_t H5Z_COUNTER[1] = {{
    H5Z_CLASS_T_VERS,           /* H5Z_class_t version          */
    FILTER_COUNTER,             /* Filter id number             */
    1, 1,                       /* Encoding and decoding enabled */
    "counter",                  /* Filter name for debugging    */
    NULL,                       /* The "can apply" callback     */
    NULL,                       /* The "set local" callback     */
    counter,                    /* The actual filter function   */
}};

/*-------------------------------------------------------------------------
 *      Count number of bytes but don't do anything else.  Keep
 *      track of the data of chunks being read from file into memory.
 */
static size_t
counter (unsigned flags, size_t cd_nelmts,
         const unsigned *cd_values, size_t nbytes,
         size_t *buf_size, void **buf)
{
    nbytes_global += nbytes;
    return nbytes;
}

/*---------------------------------------------------------------------------*/
double retrieve_time(void)
{
#ifdef H5_HAVE_GETTIMEOFDAY
    struct timeval t;
    HDgettimeofday(&t, NULL);
    return ((double)t.tv_sec + (double)t.tv_usec / 1000000);
#else
    return 0.0;
#endif
}

/*---------------------------------------------------------------------------*/
static void
cleanup (void)
{
    if (!getenv ("HDF5_NOCLEANUP")) {
        remove (FILENAME);
    }
}

/*-------------------------------------------------------------------------------
 *      Create a chunked dataset with partial chunks along either dimensions:
 *          dataset dimension:  9000 x 9
 *          chunk dimension:    2000 x 2
 */
static int create_dset1(hid_t file)
{
    hid_t        dataspace = H5I_INVALID_HID, dataset = H5I_INVALID_HID;
    hid_t        dcpl = H5I_INVALID_HID;
    hsize_t      dims[RANK]  = {DSET1_DIM1, DSET1_DIM2};
    hsize_t      chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2};
    int          data[DSET1_DIM1][DSET1_DIM2];    /* data for writing */
    int          i, j;
    
    /* Create the data space. */
    if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0)
        goto error;
    
    /* Modify dataset creation properties, i.e. enable chunking  */
    if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0)
        goto error;
    if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0)
        goto error;
    
    /* Set the dummy filter simply for counting the number of bytes being read into the memory */
    if(H5Zregister(H5Z_COUNTER) < 0)
        goto error;
    
    if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0)
        goto error;
    
    /* Create a new dataset within the file using chunk creation properties.  */
    if((dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace,
                          H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
        goto error;
    
    for (i = 0; i < DSET1_DIM1; i++)
        for (j = 0; j < DSET1_DIM2; j++)
            data[i][j] = i+j;
    
    /* Write data to dataset */
    if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
                       H5P_DEFAULT, data) < 0)
        goto error;
    
    /* Close resources */
    H5Dclose (dataset);
    H5Pclose (dcpl);
    H5Sclose (dataspace);
    return 0;
    
error:
    H5E_BEGIN_TRY {
        H5Dclose (dataset);
        H5Pclose (dcpl);
        H5Sclose (dataspace);
    } H5E_END_TRY;
    
    return 1;
}

/*---------------------------------------------------------------------------
 *      Create a chunked dataset for testing hash values:
 *          dataset dimensions: 300 x 600
 *          chunk dimensions:   100 x 100
 */
static int create_dset2(hid_t file)
{
    hid_t        dataspace = H5I_INVALID_HID, dataset = H5I_INVALID_HID;
    hid_t        dcpl = H5I_INVALID_HID;
    hsize_t      dims[RANK]  = {DSET2_DIM1, DSET2_DIM2};
    hsize_t      chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2};
    int          data[DSET2_DIM1][DSET2_DIM2];    /* data for writing */
    int          i, j;
    
    /* Create the data space. */
    if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0)
        goto error;
    
    /* Modify dataset creation properties, i.e. enable chunking  */
    if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0)
        goto error;
    if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0)
        goto error;
    
    /* Set the dummy filter simply for counting the number of bytes being read into the memory */
    if(H5Zregister(H5Z_COUNTER) < 0)
        goto error;
    if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0)
        goto error;
    
    /* Create a new dataset within the file using chunk creation properties.  */
    if((dataset = H5Dcreate2 (file, DSET2_NAME, H5T_NATIVE_INT, dataspace,
                          H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
        goto error;
    
    for (i = 0; i < DSET2_DIM1; i++)
        for (j = 0; j < DSET2_DIM2; j++)
            data[i][j] = i+j;
    
    /* Write data to dataset */
    if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
                       H5P_DEFAULT, data) < 0)
        goto error;
    
    /* Close resources */
    H5Dclose (dataset);
    H5Pclose (dcpl);
    H5Sclose (dataspace);
    
    return 0;
    
error:
    H5E_BEGIN_TRY {
        H5Dclose (dataset);
        H5Pclose (dcpl);
        H5Sclose (dataspace);
    } H5E_END_TRY;
    
    return 1;
}
/*---------------------------------------------------------------------------
 *      Check the performance of the chunk cache when partial chunks exist
 *      along the dataset dimensions.
 */
static int check_partial_chunks_perf(hid_t file)
{
    hid_t        dataset = H5I_INVALID_HID;
    hid_t        filespace = H5I_INVALID_HID;
    hid_t        memspace = H5I_INVALID_HID;
    hid_t        dapl = H5I_INVALID_HID;
    
    int          rdata[DSET1_DIM2];   /* data for reading */
    int          i;
    
    hsize_t      row_rank = 1;
    hsize_t      row_dim[1] = {DSET1_DIM2};
    hsize_t      start[RANK] = {0, 0};
    hsize_t      count[RANK] = {1, DSET1_DIM2};
    double       start_t, end_t;
    
    if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
        goto error;
    if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0)
        goto error;

    dataset = H5Dopen2 (file, DSET1_NAME, dapl);
    
    memspace = H5Screate_simple(row_rank, row_dim, NULL);
    filespace = H5Dget_space(dataset);
    
    nbytes_global = 0;
    
    start_t = retrieve_time();
    
    /* Read the data row by row */
    for(i = 0; i < DSET1_DIM1; i++) {
        start[0] = i;
        if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET,
                                     start, NULL, count, NULL) < 0)
            goto error;
        
        if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace,
                          H5P_DEFAULT, rdata) < 0)
            goto error;
    }
    
    end_t = retrieve_time();
    
#ifdef H5_HAVE_GETTIMEOFDAY
    printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global);
#else
    printf("1. Partial chunks: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global);
#endif
    
    H5Dclose (dataset);
    H5Sclose (filespace);
    H5Sclose (memspace);
    H5Pclose (dapl);
    
    return 0;
error:
    H5E_BEGIN_TRY {
        H5Dclose (dataset);
        H5Sclose (filespace);
        H5Sclose (memspace);
        H5Pclose (dapl);
    } H5E_END_TRY;
    return 1;
}

/*---------------------------------------------------------------------------
 *      Check the performance of chunk cache when the number of cache slots
 *      is smaller than the number of chunks along the fastest-growing
 *      dimension of the dataset.
 */
static int check_hash_value_perf(hid_t file)
{
    hid_t        dataset = H5I_INVALID_HID;
    hid_t        filespace = H5I_INVALID_HID;
    hid_t        memspace = H5I_INVALID_HID;
    hid_t        dapl = H5I_INVALID_HID;
    
    int          rdata[DSET2_DIM1];   /* data for reading */
    int          i;
    
    hsize_t      column_rank = 1;
    hsize_t      column_dim[1] = {DSET2_DIM1};
    hsize_t      start[RANK] = {0, 0};
    hsize_t      count[RANK] = {DSET2_DIM1, 1};
    double       start_t, end_t;
    
    if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
        goto error;
    if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0)
        goto error;
    
    if((dataset = H5Dopen2 (file, DSET2_NAME, dapl)) < 0)
        goto error;
    if((memspace = H5Screate_simple(column_rank, column_dim, NULL)) < 0)
        goto error;
    if((filespace = H5Dget_space(dataset)) < 0)
        goto error;
    
    nbytes_global = 0;
    
    start_t = retrieve_time();
    
    /* Read the data column by column */
    for(i = 0; i < DSET2_DIM2; i++) {
        start[1] = i;
        if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET,
                                     start, NULL, count, NULL) < 0)
            goto error;
        
        if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace,
                          H5P_DEFAULT, rdata) < 0)
            goto error;
    }
    
    end_t = retrieve_time();
    
#ifdef H5_HAVE_GETTIMEOFDAY
    printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global);
#else
    printf("2. Hash value: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global);
#endif
    
    H5Dclose (dataset);
    H5Sclose (filespace);
    H5Sclose (memspace);
    H5Pclose (dapl);
    return 0;
    
error:
    H5E_BEGIN_TRY {
        H5Dclose (dataset);
        H5Sclose (filespace);
        H5Sclose (memspace);
        H5Pclose (dapl);
    } H5E_END_TRY;
    return 1;
}

/*-------------------------------------------------------------------------------------
 *  Purpose: check the performance of chunk cache in these two cases (HDFFV-10601):
 *          1. partial chunks exist along any dimension.
 *          2. number of slots in chunk cache is smaller than the number of chunks
 *             in the fastest-growing dimension.
 *-------------------------------------------------------------------------------------*/
int
main (void)
{
    hid_t        file;                          /* handles */
    int          nerrors = 0;
    
    /* Create a new file. If file exists its contents will be overwritten. */
    if((file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
        goto error;

    nerrors += create_dset1(file);
    nerrors += create_dset2(file);
    
    if(H5Fclose (file) < 0)
        goto error;

    /* Re-open the file for testing performance. */
    if((file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
        goto error;

    nerrors += check_partial_chunks_perf(file);
    nerrors += check_hash_value_perf(file);
    
    if(H5Fclose (file) < 0)
        goto error;
    
    if (nerrors>0) goto error;
    cleanup();
    return 0;
    
error:
    fprintf(stderr, "*** ERRORS DETECTED ***\n");
    return 1;
}