path: root/tools/src/h5repack/h5repack_verify.c

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Board of Trustees of the University of Illinois.         *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the root of the source code       *
 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
 * If you do not have access to either file, you may request a copy from     *
 * help@hdfgroup.org.                                                        *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#include "h5repack.h"
#include "h5tools.h"
#include "h5tools_utils.h"

/* number of members in an array */
#ifndef NELMTS
#define NELMTS(X) (sizeof(X) / sizeof(X[0]))
#endif

static int verify_layout(hid_t pid, pack_info_t *obj);
static int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter);

/*-------------------------------------------------------------------------
 * Function: h5repack_verify
 *
 * Purpose: verify if filters and layout in the input file match the output file
 *
 * Return:
 *  1 match
 *  0 do not match
 * -1 error
 *-------------------------------------------------------------------------
 */

int
h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options)
{
    hid_t                 fidin  = H5I_INVALID_HID; /* file ID for input file*/
    hid_t                 fidout = H5I_INVALID_HID; /* file ID for output file*/
    hid_t                 did    = H5I_INVALID_HID; /* dataset ID */
    hid_t                 pid    = H5I_INVALID_HID; /* dataset creation property list ID */
    hid_t                 sid    = H5I_INVALID_HID; /* space ID */
    hid_t                 tid    = H5I_INVALID_HID; /* type ID */
    int                   ok     = 1;               /* step results */
    unsigned int          i;
    trav_table_t *        travt    = NULL;
    hid_t                 fcpl_in  = H5I_INVALID_HID;  /* file creation property for input file */
    hid_t                 fcpl_out = H5I_INVALID_HID;  /* file creation property for output file */
    H5F_fspace_strategy_t in_strategy, out_strategy;   /* file space handling strategy for in/output file */
    hbool_t               in_persist, out_persist;     /* free-space persist status for in/output file */
    hsize_t               in_threshold, out_threshold; /* free-space section threshold for in/output file */
    hsize_t               in_pagesize, out_pagesize;   /* file space page size for input/output file */
    int                   ret_value = 0;

    /* open the output file */
    if ((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Fopen failed on <%s>", out_fname);

    for (i = 0; i < options->op_tbl->nelems; i++) {
        char *       name = options->op_tbl->objs[i].path;
        pack_info_t *obj  = &options->op_tbl->objs[i];

        /*-------------------------------------------------------------------------
         * open
         *-------------------------------------------------------------------------
         */
        if ((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed on <%s>", name);
        if ((sid = H5Dget_space(did)) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Dget_space failed");
        if ((pid = H5Dget_create_plist(did)) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Dget_create_plist failed");
        if ((tid = H5Dget_type(did)) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Dget_type failed");

        /*-------------------------------------------------------------------------
         * filter check
         *-------------------------------------------------------------------------
         */
        if (verify_filters(pid, tid, obj->nfilters, obj->filter) <= 0)
            ok = 0;

        /*-------------------------------------------------------------------------
         * layout check
         *-------------------------------------------------------------------------
         */
        if ((obj->layout != -1) && (verify_layout(pid, obj) == 0))
            ok = 0;

        /*-------------------------------------------------------------------------
         * close
         *-------------------------------------------------------------------------
         */
        if (H5Pclose(pid) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
        if (H5Sclose(sid) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Sclose failed");
        if (H5Dclose(did) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed");
        if (H5Tclose(tid) < 0)
            H5TOOLS_GOTO_ERROR((-1), "H5Tclose failed");
    }

    /*-------------------------------------------------------------------------
     * check for the "all" objects option
     *-------------------------------------------------------------------------
     */

    if (options->all_filter == 1 || options->all_layout == 1) {
        /* Initialize indexing options */
        h5trav_set_index(sort_by, sort_order);
        /* init table */
        trav_table_init(fidout, &travt);

        /* get the list of objects in the file */
        if (h5trav_gettable(fidout, travt) < 0)
            H5TOOLS_GOTO_ERROR((-1), "h5trav_gettable failed");

        for (i = 0; i < travt->nobjs; i++) {
            char *name = travt->objs[i].name;

            if (travt->objs[i].type == H5TRAV_TYPE_DATASET) {
                /*-------------------------------------------------------------------------
                 * open
                 *-------------------------------------------------------------------------
                 */
                if ((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed on <%s>", name);
                if ((sid = H5Dget_space(did)) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Dget_space failed");
                if ((pid = H5Dget_create_plist(did)) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Dget_create_plist failed");
                if ((tid = H5Dget_type(did)) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Dget_type failed");

                /*-------------------------------------------------------------------------
                 * filter check
                 *-------------------------------------------------------------------------
                 */
                if (options->all_filter == 1) {
                    if (verify_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0)
                        ok = 0;
                }

                /*-------------------------------------------------------------------------
                 * layout check
                 *-------------------------------------------------------------------------
                 */
                if (options->all_layout == 1) {
                    pack_info_t pack;

                    init_packobject(&pack);
                    pack.layout = options->layout_g;
                    pack.chunk  = options->chunk_g;
                    if (verify_layout(pid, &pack) == 0)
                        ok = 0;
                }

                /*-------------------------------------------------------------------------
                 * close
                 *-------------------------------------------------------------------------
                 */
                if (H5Pclose(pid) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
                if (H5Sclose(sid) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Sclose failed");
                if (H5Dclose(did) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed");
                if (H5Tclose(tid) < 0)
                    H5TOOLS_GOTO_ERROR((-1), "H5Tclose failed");
            } /* if */
        }     /* i */

        /* free table */
        trav_table_free(travt);
        travt = NULL;
    }

    /*-------------------------------------------------------------------------
     * Verify that file space info are set as expected
     *-------------------------------------------------------------------------
     */

    /* open the input file */
    if ((fidin = H5Fopen(in_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Fopen failed on file <%s>", in_fname);

    /* Get file creation property list for input file */
    if ((fcpl_in = H5Fget_create_plist(fidin)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Fget_create_plist failed to retrieve file creation property list");

    /* Get file space info for input file */
    if (H5Pget_file_space_strategy(fcpl_in, &in_strategy, &in_persist, &in_threshold) < 0)
        H5TOOLS_GOTO_ERROR((-1),
                           "H5Pget_file_space_strategy failed to retrieve file space strategy & threshold");

    /* Get file space page size for input file */
    if (H5Pget_file_space_page_size(fcpl_in, &in_pagesize) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Pget_file_space_page_size failed to retrieve file space page size");

    /* Output file is already opened */
    /* Get file creation property list for output file */
    if ((fcpl_out = H5Fget_create_plist(fidout)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Fget_create_plist failed to retrieve file creation property list");

    /* Get file space info for output file */
    if (H5Pget_file_space_strategy(fcpl_out, &out_strategy, &out_persist, &out_threshold) < 0)
        H5TOOLS_GOTO_ERROR((-1),
                           "H5Pget_file_space_strategy failed to retrieve file space strategy & threshold");

    /* Get file space page size for output file */
    if (H5Pget_file_space_page_size(fcpl_out, &out_pagesize) < 0)
        H5TOOLS_GOTO_ERROR((-1), "H5Pget_file_space_page_size failed to retrieve file space page size");

    /*
     * If -S option is set, the file space handling strategy should be set as specified.
     * If -S option is not set, the file space handling strategy should be
     * the same as the input file's strategy.
     */
    if (options->fs_strategy) {
        if (out_strategy != (options->fs_strategy == (H5F_fspace_strategy_t)-1 ? 0 : options->fs_strategy))
            H5TOOLS_GOTO_ERROR((-1), "file space strategy not set as unexpected");
    }
    else {
        if (out_strategy != in_strategy)
            H5TOOLS_GOTO_ERROR((-1), "file space strategy not set as unexpected");
    }

    /*
     * If the -P option is set, the free-space persist status should be set as specified.
     * If the -P option is not set, the free-space persist status should be
     * the same as the input file's free-space persist status
     */
    if (options->fs_persist) {
        if (out_persist != (hbool_t)(options->fs_persist == (-1) ? FALSE : options->fs_persist))
            H5TOOLS_GOTO_ERROR((-1), "free-space persist status not set as unexpected");
    }
    else {
        if (out_persist != in_persist)
            H5TOOLS_GOTO_ERROR((-1), "free-space persist status not set as unexpected");
    }

    /*
     * If the -T option is set, the threshold size should be set as specified.
     * If the -T option is not set, the threshold should be the same as the
     * input file's threshold size.
     */
    if (options->fs_threshold) {
        if (out_threshold != (hsize_t)(options->fs_threshold == (-1) ? 0 : options->fs_threshold))
            H5TOOLS_GOTO_ERROR((-1), "threshold not set as unexpected");
    }
    else {
        if (out_threshold != in_threshold)
            H5TOOLS_GOTO_ERROR((-1), "threshold not set as unexpected");
    }

    /*
     * If the -G option is set, the file space page size should be set as specified.
     * If the -G option is not set, the file space page size should be
     * the same as the input file's file space page size.
     */
    if (options->fs_pagesize) {
        if (out_pagesize != (hsize_t)(options->fs_pagesize == (-1) ? 0 : options->fs_pagesize))
            H5TOOLS_GOTO_ERROR((-1), "file space page size not set as unexpected");
    }
    else { /* "-G" is not set */
        if (out_pagesize != in_pagesize)
            H5TOOLS_GOTO_ERROR((-1), "file space page size not set as unexpected");
    }

    ret_value = ok;

done:
    H5E_BEGIN_TRY
    {
        H5Pclose(fcpl_in);
        H5Pclose(fcpl_out);
        H5Pclose(pid);
        H5Sclose(sid);
        H5Dclose(did);
        H5Tclose(tid);
        H5Fclose(fidin);
        H5Fclose(fidout);
        if (travt)
            trav_table_free(travt);
    }
    H5E_END_TRY;

    return ret_value;
} /* h5repack_verify() */

/*-------------------------------------------------------------------------
 * Function: verify_layout
 *
 * Purpose:  verify which layout is present in the property list DCPL_ID
 *           H5D_COMPACT     = 0
 *           H5D_CONTIGUOUS  = 1
 *           H5D_CHUNKED     = 2
 *
 * Return:
 *           1 has,
 *           0 does not,
 *          -1 error
 *-------------------------------------------------------------------------
 */

int
verify_layout(hid_t pid, pack_info_t *obj)
{
    hsize_t      chsize[64]; /* chunk size in elements */
    H5D_layout_t layout;     /* layout */
    int          nfilters;   /* number of filters */
    int          rank;       /* rank */
    int          i;          /* index */

    /* check if we have filters in the input object */
    if ((nfilters = H5Pget_nfilters(pid)) < 0)
        return -1;

    /* a non chunked layout was requested on a filtered object */
    if (nfilters && obj->layout != H5D_CHUNKED)
        return 0;

    /* get layout */
    if ((layout = H5Pget_layout(pid)) < 0)
        return -1;

    if (obj->layout != layout)
        return 0;

    if (layout == H5D_CHUNKED) {
        if ((rank = H5Pget_chunk(pid, NELMTS(chsize), chsize /*out*/)) < 0)
            return -1;
        if (obj->chunk.rank != rank)
            return 0;
        for (i = 0; i < rank; i++)
            if (chsize[i] != obj->chunk.chunk_lengths[i])
                return 0;
    }

    return 1;
}

/*-------------------------------------------------------------------------
 * Function: h5repack_cmp_pl
 *
 * Purpose:  compare 2 files for identical property lists of all objects
 *
 * Return:
 *           1 = identical,
 *           0 = not identical,
 *          -1 = error
 *-------------------------------------------------------------------------
 */

int
h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t fname2_fapl)
{
    hid_t         fid1   = H5I_INVALID_HID; /* file ID */
    hid_t         fid2   = H5I_INVALID_HID; /* file ID */
    hid_t         dset1  = H5I_INVALID_HID; /* dataset ID */
    hid_t         dset2  = H5I_INVALID_HID; /* dataset ID */
    hid_t         gid    = H5I_INVALID_HID; /* group ID */
    hid_t         dcpl1  = H5I_INVALID_HID; /* dataset creation property list ID */
    hid_t         dcpl2  = H5I_INVALID_HID; /* dataset creation property list ID */
    hid_t         gcplid = H5I_INVALID_HID; /* group creation property list */
    unsigned      crt_order_flag1;          /* group creation order flag */
    unsigned      crt_order_flag2;          /* group creation order flag */
    trav_table_t *trav = NULL;
    unsigned int  i;
    int           ret_value = 1;

    /*-------------------------------------------------------------------------
     * open the files
     *-------------------------------------------------------------------------
     */
    /* Open the files */
    if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, TRUE, NULL, 0)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname1, H5FOPENERROR);
    if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, TRUE, NULL, 0)) < 0)
        H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname2, H5FOPENERROR);

    /*-------------------------------------------------------------------------
     * get file table list of objects
     *-------------------------------------------------------------------------
     */
    /* Initialize indexing options */
    h5trav_set_index(sort_by, sort_order);
    /* init table */
    trav_table_init(fid1, &trav);
    if (h5trav_gettable(fid1, trav) < 0)
        H5TOOLS_GOTO_ERROR((-1), "h5trav_gettable failed");

    /*-------------------------------------------------------------------------
     * traverse the suppplied object list
     *-------------------------------------------------------------------------
     */
    for (i = 0; i < trav->nobjs; i++) {
        if (trav->objs[i].type == H5TRAV_TYPE_GROUP) {
            if ((gid = H5Gopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gopen2 failed on first <%s>", trav->objs[i].name);
            if ((gcplid = H5Gget_create_plist(gid)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gget_create_plist failed");
            if (H5Pget_link_creation_order(gcplid, &crt_order_flag1) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pget_link_creation_order failed");
            if (H5Pclose(gcplid) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
            if (H5Gclose(gid) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gclose failed");

            if ((gid = H5Gopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gopen2 failed on second <%s>", trav->objs[i].name);
            if ((gcplid = H5Gget_create_plist(gid)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gget_create_plist failed");
            if (H5Pget_link_creation_order(gcplid, &crt_order_flag2) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pget_link_creation_order failed");
            if (H5Pclose(gcplid) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
            if (H5Gclose(gid) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Gclose failed");

            if (crt_order_flag1 != crt_order_flag2)
                H5TOOLS_GOTO_ERROR(0, "property lists failed for <%s> are different", trav->objs[i].name);
        }
        else if (trav->objs[i].type == H5TRAV_TYPE_DATASET) {
            if ((dset1 = H5Dopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed on first <%s>", trav->objs[i].name);
            if ((dset2 = H5Dopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dopen2 failed on second <%s>", trav->objs[i].name);
            if ((dcpl1 = H5Dget_create_plist(dset1)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dget_create_plist failed");
            if ((dcpl2 = H5Dget_create_plist(dset2)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dget_create_plist failed");

            /*-------------------------------------------------------------------------
             * compare the property lists
             *-------------------------------------------------------------------------
             */
            if ((ret_value = H5Pequal(dcpl1, dcpl2)) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pequal failed");

            if (ret_value == 0)
                H5TOOLS_GOTO_ERROR(0, "property lists failed for <%s> are different", trav->objs[i].name);

            /*-------------------------------------------------------------------------
             * close
             *-------------------------------------------------------------------------
             */
            if (H5Pclose(dcpl1) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
            if (H5Pclose(dcpl2) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
            if (H5Dclose(dset1) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed");
            if (H5Dclose(dset2) < 0)
                H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed");
        } /*if*/
    }     /*for*/

done:
    H5E_BEGIN_TRY
    {
        H5Pclose(dcpl1);
        H5Pclose(dcpl2);
        H5Dclose(dset1);
        H5Dclose(dset2);
        H5Fclose(fid1);
        H5Fclose(fid2);
        H5Pclose(gcplid);
        H5Gclose(gid);
        if (trav)
            trav_table_free(trav);
    }
    H5E_END_TRY;

    return ret_value;
}

/*-------------------------------------------------------------------------
 * Function: verify_filters
 *
 * Purpose:  verify if all requested filters in the array FILTER obtained
 *           from user input are present in the property list PID obtained from
 *           the output file
 *
 * Return:
 *           1 match
 *           0 do not match
 *          -1 error
 *-------------------------------------------------------------------------
 */

static int
verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter)
{
    int          nfilters_dcpl; /* number of filters in DCPL*/
    unsigned     filt_flags;    /* filter flags */
    H5Z_filter_t filtn;         /* filter identification number */
    unsigned     cd_values[20]; /* filter client data values */
    size_t       cd_nelmts;     /* filter client number of values */
    char         f_name[256];   /* filter name */
    size_t       size;          /* type size */
    int          i;             /* index */
    unsigned     j;             /* index */

    /* get information about filters */
    if ((nfilters_dcpl = H5Pget_nfilters(pid)) < 0)
        return -1;

    /* if we do not have filters and the requested filter is NONE, return 1 */
    if (!nfilters_dcpl && nfilters == 1 && filter[0].filtn == H5Z_FILTER_NONE)
        return 1;

    /* else the numbers of filters must match */
    if (nfilters_dcpl != nfilters)
        return 0;

    /*-------------------------------------------------------------------------
     * build a list with DCPL filters
     *-------------------------------------------------------------------------
     */

    for (i = 0; i < nfilters_dcpl; i++) {
        cd_nelmts = NELMTS(cd_values);
        filtn = H5Pget_filter2(pid, (unsigned)i, &filt_flags, &cd_nelmts, cd_values, sizeof(f_name), f_name,
                               NULL);

        /* filter ID */
        if (filtn < 0)
            return -1;
        if (filtn != filter[i].filtn)
            return 0;

        /* compare client data values. some filters do return local values */
        switch (filtn) {
            case H5Z_FILTER_NONE:
                break;

            case H5Z_FILTER_SHUFFLE:
                /* 1 private client value is returned by DCPL */
                if (cd_nelmts != H5Z_SHUFFLE_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SHUFFLE_USER_NPARMS)
                    return 0;

                /* get dataset's type size */
                if ((size = H5Tget_size(tid)) <= 0)
                    return -1;

                /* the private client value holds the dataset's type size */
                if (size != cd_values[0])
                    return 0;

                break;

            case H5Z_FILTER_SZIP:
                /* 4 private client values are returned by DCPL */
                if (cd_nelmts != H5Z_SZIP_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SZIP_USER_NPARMS)
                    return 0;

                /* "User" parameter for pixels-per-block (index 1) */
                if (cd_values[H5Z_SZIP_PARM_PPB] != filter[i].cd_values[H5Z_SZIP_PARM_PPB])
                    return 0;

                break;

            case H5Z_FILTER_NBIT:
                /* only client data values number of values checked */
                if (H5Z_NBIT_USER_NPARMS != filter[i].cd_nelmts)
                    return 0;
                break;

            case H5Z_FILTER_SCALEOFFSET:
                /* only client data values checked */
                for (j = 0; j < H5Z_SCALEOFFSET_USER_NPARMS; j++)
                    if (cd_values[j] != filter[i].cd_values[j])
                        return 0;
                break;

            /* for these filters values must match, no local values set in DCPL */
            case H5Z_FILTER_FLETCHER32:
            case H5Z_FILTER_DEFLATE:

                if (cd_nelmts != filter[i].cd_nelmts)
                    return 0;

                for (j = 0; j < cd_nelmts; j++)
                    if (cd_values[j] != filter[i].cd_values[j])
                        return 0;

                break;

            default:
                if (cd_nelmts != filter[i].cd_nelmts)
                    return 0;

                for (j = 0; j < cd_nelmts; j++)
                    if (cd_values[j] != filter[i].cd_values[j])
                        return 0;
                break;

        } /* switch */
    }

    return 1;
}