diff options
Diffstat (limited to 'tools/src/h5repack')
| -rw-r--r-- | tools/src/h5repack/CMakeLists.txt | 50 | ||||
| -rw-r--r-- | tools/src/h5repack/Makefile.am | 42 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack.c | 942 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack.h | 243 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_copy.c | 1581 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_filters.c | 493 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_main.c | 664 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_opttable.c | 367 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_parse.c | 616 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_refs.c | 877 | ||||
| -rw-r--r-- | tools/src/h5repack/h5repack_verify.c | 675 |
11 files changed, 6550 insertions, 0 deletions
diff --git a/tools/src/h5repack/CMakeLists.txt b/tools/src/h5repack/CMakeLists.txt new file mode 100644 index 0000000..cb7f5f6 --- /dev/null +++ b/tools/src/h5repack/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required (VERSION 3.1.0) +PROJECT (HDF5_TOOLS_SRC_H5REPACK) + +#----------------------------------------------------------------------------- +# Setup include Directories +#----------------------------------------------------------------------------- +INCLUDE_DIRECTORIES (${HDF5_TOOLS_DIR}/lib) +INCLUDE_DIRECTORIES (${HDF5_TEST_SRC_DIR}) + +# -------------------------------------------------------------------- +# Add h5Repack executables +# -------------------------------------------------------------------- +set (REPACK_COMMON_SOURCES + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_copy.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_filters.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_opttable.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_parse.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_refs.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_verify.c + ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack.c +) + +add_executable (h5repack ${REPACK_COMMON_SOURCES} ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_main.c) +TARGET_NAMING (h5repack STATIC) +TARGET_C_PROPERTIES (h5repack STATIC " " " ") +target_link_libraries (h5repack ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +set_target_properties (h5repack PROPERTIES FOLDER tools) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5repack") + +set (H5_DEP_EXECUTABLES h5repack) + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- + +#INSTALL_PROGRAM_PDB (h5repack ${HDF5_INSTALL_BIN_DIR} toolsapplications) + +install ( + TARGETS + h5repack + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications +) diff --git a/tools/src/h5repack/Makefile.am b/tools/src/h5repack/Makefile.am new file mode 100644 index 0000000..e6e5a56 --- /dev/null +++ b/tools/src/h5repack/Makefile.am @@ -0,0 +1,42 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src, test, and tools/lib directories +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib + +# Our main target, h5repack tool +bin_PROGRAMS=h5repack + +# Add h5repack specific linker flags here +h5repack_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) + +# Depend on the hdf5 library, the tools library, the test library +LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) + +# Source files +COMMON_SOURCES=h5repack.c h5repack_copy.c h5repack_filters.c \ + h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ + h5repack_verify.c + +h5repack_SOURCES=$(COMMON_SOURCES) h5repack_main.c + +include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c new file mode 100644 index 0000000..ef2085c --- /dev/null +++ b/tools/src/h5repack/h5repack.c @@ -0,0 +1,942 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +#include "H5private.h" +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/*------------------------------------------------------------------------- + * File: h5repack.c + * Purpose: Public API functions + *------------------------------------------------------------------------- + */ + +static int check_options(pack_opt_t *options); +static int check_objects(const char* fname, pack_opt_t *options); +static const char* get_sfilter(H5Z_filter_t filtn); +static int have_request(pack_opt_t *options); + +/*------------------------------------------------------------------------- + * Function: h5repack + * + * Purpose: locate all high-level HDF5 objects in the file + * and compress/chunk them using options + * + * Algorithm: 2 traversals are made to the file; the 1st builds a list of + * the objects, the 2nd makes a copy of them, using the options; + * the reason for the 1st traversal is to check for invalid + * object name requests + * + * Return: 0, ok, -1, fail + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +int h5repack(const char* infile, const char* outfile, pack_opt_t *options) { + /* check input */ + if (check_options(options) < 0) + return -1; + + /* check for objects in input that are in the file */ + if (check_objects(infile, options) < 0) + return -1; + + /* copy the objects */ + if (copy_objects(infile, outfile, options) < 0) + return -1; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: h5repack_init + * + * Purpose: initialize options + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, + H5F_file_space_type_t strategy, hsize_t threshold) { + int k, n; + + HDmemset(options, 0, sizeof(pack_opt_t)); + options->min_comp = 0; + options->verbose = verbose; + options->latest = latest; + options->layout_g = H5D_LAYOUT_ERROR; + + for (n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { + options->filter_g[n].filtn = -1; + options->filter_g[n].cd_nelmts = 0; + for (k = 0; k < CD_VALUES; k++) + options->filter_g[n].cd_values[k] = 0; + } + + options->fs_strategy = strategy; + options->fs_threshold = threshold; + + return (options_table_init(&(options->op_tbl))); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_end + * + * Purpose: free options table + * + *------------------------------------------------------------------------- + */ + +int h5repack_end(pack_opt_t *options) { + return options_table_free(options->op_tbl); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_addfilter + * + * Purpose: add a compression -f option to table + * Example: -f dset:GZIP=6 + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int +h5repack_addfilter(const char* str, pack_opt_t *options) +{ + obj_list_t *obj_list = NULL; /* one object list for the -f and -l option entry */ + filter_info_t filter; /* filter info for the current -f option entry */ + unsigned n_objs; /* number of objects in the current -f or -l option entry */ + int is_glb; /* is the filter global */ + + /* parse the -f option */ + if (NULL == (obj_list = parse_filter(str, &n_objs, &filter, options, &is_glb))) + return -1; + + /* if it applies to all objects */ + if (is_glb) { + int n; + + n = options->n_filter_g++; /* increase # of global filters */ + if (options->n_filter_g > H5_REPACK_MAX_NFILTERS) { + error_msg("maximum number of filters exceeded for <%s>\n", str); + HDfree(obj_list); + return -1; + } + + options->filter_g[n] = filter; + } + else + options_add_filter(obj_list, n_objs, filter, options->op_tbl); + + HDfree(obj_list); + return 0; +} + +/*------------------------------------------------------------------------- + * Function: h5repack_addlayout + * + * Purpose: add a layout option + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int +h5repack_addlayout(const char* str, pack_opt_t *options) +{ + obj_list_t *obj_list = NULL; /*one object list for the -t and -c option entry */ + unsigned n_objs; /*number of objects in the current -t or -c option entry */ + pack_info_t pack; /*info about layout to extract from parse */ + int j; + int ret_value = -1; + + init_packobject(&pack); + + if (options->all_layout == 1) { + error_msg( "invalid layout input: 'all' option is present with other objects <%s>\n", str); + return ret_value; + } + + /* parse the layout option */ + obj_list = parse_layout(str, &n_objs, &pack, options); + if (obj_list) { + /* set layout option */ + options->layout_g = pack.layout; + + /* no individual dataset specified */ + if (options->all_layout == 1) { + if (pack.layout == H5D_CHUNKED) { + /* -2 means the NONE option, remove chunking + and set the global layout to contiguous */ + if (pack.chunk.rank == -2) + options->layout_g = H5D_CONTIGUOUS; + /* otherwise set the global chunking type */ + else { + options->chunk_g.rank = pack.chunk.rank; + for (j = 0; j < pack.chunk.rank; j++) + options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; + } + } + } + + /* individual dataset specified */ + if (options->all_layout == 0) + ret_value = options_add_layout(obj_list, n_objs, &pack, options->op_tbl); + + HDfree(obj_list); + ret_value = 0; + } + + return ret_value; +} + +/* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() + * were located in h5repack_copy.c as static prior to bugfix1726. + * Made shared functions as copy_attr() was needed in h5repack_refs.c. + * However copy_attr() may be obsoleted when H5Acopy is available and put back + * others to static in h5repack_copy.c. + */ +/*------------------------------------------------------------------------- + * Function: copy_named_datatype + * + * Purpose: Copies the specified datatype anonymously, and returns an open + * id for that datatype in the output file. The first time this + * is called it scans every named datatype in travt into a + * private stack, afterwards it simply scans that stack. The id + * returned must be closed after it is no longer needed. + * named_datatype_free must be called before the program exits + * to free the stack. + * + * Programmer: Neil Fortner + * + * Date: April 14, 2009 + * + *------------------------------------------------------------------------- + */ +hid_t copy_named_datatype(hid_t type_in, hid_t fidout, + named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) { + named_dt_t *dt = *named_dt_head_p; /* Stack pointer */ + named_dt_t *dt_ret = NULL; /* Datatype to return */ + H5O_info_t oinfo; /* Object info of input dtype */ + hid_t ret_value = -1; /* The identifier of the named dtype in the out file */ + + if (H5Oget_info(type_in, &oinfo) < 0) + goto done; + + if (*named_dt_head_p) { + /* Stack already exists, search for the datatype */ + while (dt && dt->addr_in != oinfo.addr) + dt = dt->next; + + dt_ret = dt; + } + else { + /* Create the stack */ + size_t i; + + for (i = 0; i < travt->nobjs; i++) { + if (travt->objs[i].type == H5TRAV_TYPE_NAMED_DATATYPE) { + /* Push onto the stack */ + if (NULL == (dt = (named_dt_t *) HDmalloc(sizeof(named_dt_t)))) { + goto done; + } + dt->next = *named_dt_head_p; + *named_dt_head_p = dt; + + /* Update the address and id */ + dt->addr_in = travt->objs[i].objno; + dt->id_out = -1; + + /* Check if this type is the one requested */ + if (oinfo.addr == dt->addr_in) { + HDassert(!dt_ret); + dt_ret = dt; + } /* end if */ + } /* end if */ + } /* end for */ + } /* end else */ + + /* Handle the case that the requested datatype was not found. This is + * possible if the datatype was committed anonymously in the input file. */ + if (!dt_ret) { + /* Push the new datatype onto the stack */ + if (NULL == (dt_ret = (named_dt_t *) HDmalloc(sizeof(named_dt_t)))) { + goto done; + } + dt_ret->next = *named_dt_head_p; + *named_dt_head_p = dt_ret; + + /* Update the address and id */ + dt_ret->addr_in = oinfo.addr; + dt_ret->id_out = -1; + } /* end if */ + + /* If the requested datatype does not yet exist in the output file, copy it + * anonymously */ + if (dt_ret->id_out < 0) { + if (options->use_native == 1) + dt_ret->id_out = h5tools_get_native_type(type_in); + else + dt_ret->id_out = H5Tcopy(type_in); + if (dt_ret->id_out < 0) + goto done; + if (H5Tcommit_anon(fidout, dt_ret->id_out, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto done; + } /* end if */ + + /* Set return value */ + ret_value = dt_ret->id_out; + + /* Increment the ref count on id_out, because the calling function will try + * to close it */ + if(H5Iinc_ref(ret_value) < 0) { + ret_value = -1; + } + +done: + return (ret_value); +} /* end copy_named_datatype */ + +/*------------------------------------------------------------------------- + * Function: named_datatype_free + * + * Purpose: Frees the stack of named datatypes. + * + * Programmer: Neil Fortner + * + * Date: April 14, 2009 + * + *------------------------------------------------------------------------- + */ +int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err) { + named_dt_t *dt = *named_dt_head_p; + int ret_value = -1; + + while (dt) { + /* Pop the datatype off the stack and free it */ + if (H5Tclose(dt->id_out) < 0 && !ignore_err) + goto done; + dt = dt->next; + HDfree(*named_dt_head_p); + *named_dt_head_p = dt; + } /* end while */ + + ret_value = 0; + +done: + return (ret_value); +} /* end named_datatype_free */ + +/*------------------------------------------------------------------------- + * Function: copy_attr + * + * Purpose: copy attributes located in LOC_IN, which is obtained either from + * loc_id = H5Gopen2( fid, name); + * loc_id = H5Dopen2( fid, name); + * loc_id = H5Topen2( fid, name); + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 28, 2003 + * + *------------------------------------------------------------------------- + */ +int +copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, + trav_table_t *travt, pack_opt_t *options) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t attr_id = -1; /* attr ID */ + hid_t attr_out = -1; /* attr ID */ + hid_t space_id = -1; /* space ID */ + hid_t ftype_id = -1; /* file type ID */ + hid_t wtype_id = -1; /* read/write type ID */ + size_t msize; /* size of type */ + void *buf = NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + htri_t is_named; /* Whether the datatype is named */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + H5O_info_t oinfo; /* object info */ + int j; + unsigned u; + hbool_t is_ref = 0; + H5T_class_t type_class = -1; + + if (H5Oget_info(loc_in, &oinfo) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Oget_info failed"); + + /*------------------------------------------------------------------------- + * copy all attributes + *------------------------------------------------------------------------- + */ + for (u = 0; u < (unsigned) oinfo.num_attrs; u++) { + /* open attribute */ + if ((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) u, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aopen_by_idx failed"); + + /* get name */ + if (H5Aget_name(attr_id, (size_t) 255, name) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + + /* get the file datatype */ + if ((ftype_id = H5Aget_type(attr_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_type failed"); + + /* Check if the datatype is committed */ + if ((is_named = H5Tcommitted(ftype_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tcommitted failed"); + if (is_named && travt) { + hid_t fidout; + + /* Create out file id */ + if ((fidout = H5Iget_file_id(loc_out)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Iget_file_id failed"); + + /* Copy named dt */ + if ((wtype_id = copy_named_datatype(ftype_id, fidout, named_dt_head_p, travt, options)) < 0) { + H5Fclose(fidout); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_named_datatype failed"); + } /* end if */ + + if (H5Fclose(fidout) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed"); + } /* end if */ + else { + if (options->use_native == 1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); + } /* end else */ + + /* get the dataspace handle */ + if ((space_id = H5Aget_space(attr_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_space failed"); + + /* get dimensions */ + if ((rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed"); + + nelmts = 1; + for (j = 0; j < rank; j++) + nelmts *= dims[j]; + + if ((msize = H5Tget_size(wtype_id)) == 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); + + /*------------------------------------------------------------------------- + * object references are a special case. We cannot just copy the buffers, + * but instead we recreate the reference. + * This is done on a second sweep of the file that just copies the referenced + * objects at copy_refs_attr() + *------------------------------------------------------------------------- + */ + type_class = H5Tget_class(wtype_id); + is_ref = (type_class == H5T_REFERENCE); + if (type_class == H5T_VLEN || type_class == H5T_ARRAY) { + hid_t base_type = -1; + + base_type = H5Tget_super(ftype_id); + is_ref = (is_ref || (H5Tget_class(base_type) == H5T_REFERENCE)); + H5Tclose(base_type); + } + + if (type_class == H5T_COMPOUND) { + int nmembers = H5Tget_nmembers(wtype_id); + + for (j = 0; j < nmembers; j++) { + hid_t mtid = H5Tget_member_type(wtype_id, (unsigned) j); + H5T_class_t mtclass = H5Tget_class(mtid); + H5Tclose(mtid); + + if (mtclass == H5T_REFERENCE) { + is_ref = 1; + break; + } + } /* for (j=0; i<nmembers; j++) */ + } /* if (type_class == H5T_COMPOUND) */ + + if (is_ref) { + ; /* handled by copy_refs_attr() */ + } + else { + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + + buf = (void *) HDmalloc((size_t)(nelmts * msize)); + if (buf == NULL) { + error_msg("h5repack", "cannot read into memory\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + if (H5Aread(attr_id, wtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out = H5Acreate2(loc_out, name, wtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Acreate2 failed"); + if (H5Awrite(attr_out, wtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Awrite failed"); + + /*close*/ + if (H5Aclose(attr_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); + + /* Check if we have VL data and string in the attribute's datatype that must + * be reclaimed */ + if (TRUE == h5tools_detect_vlen(wtype_id)) + H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); + HDfree(buf); + buf = NULL; + } /*H5T_REFERENCE*/ + + if (options->verbose) + printf(FORMAT_OBJ_ATTR, "attr", name); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Tclose(wtype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Sclose(space_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if (H5Aclose(attr_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); + } /* u */ + + return 0; + +done: + H5E_BEGIN_TRY { + if (buf) { + /* Check if we have VL data and string in the attribute's datatype that must + * be reclaimed */ + if (TRUE == h5tools_detect_vlen(wtype_id)) + H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); + + /* Free buf */ + HDfree(buf); + } /* end if */ + + H5Tclose(ftype_id); + H5Tclose(wtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + } H5E_END_TRY; + + return ret_value; +} /* end copy_attr() */ + +/*------------------------------------------------------------------------- + * Function: check_options + * + * Purpose: print options, checks for invalid options + * + * Return: void, return -1 on error + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + * Modification: + * Peter Cao, July 9, 2007 + * Add "-L, --latest" and other options to pack a file with the latest file format + * + *------------------------------------------------------------------------- + */ +static int check_options(pack_opt_t *options) { + unsigned int i; + int k, j, has_cp = 0, has_ck = 0; + char slayout[30]; + + /*------------------------------------------------------------------------- + * objects to layout + *------------------------------------------------------------------------- + */ + if (options->verbose && have_request(options) /* only print if requested */) { + printf("Objects to modify layout are...\n"); + if (options->all_layout == 1) { + switch (options->layout_g) { + case H5D_COMPACT: + strcpy(slayout, "compact"); + break; + case H5D_CONTIGUOUS: + strcpy(slayout, "contiguous"); + break; + case H5D_CHUNKED: + strcpy(slayout, "chunked"); + break; + case H5D_VIRTUAL: + strcpy(slayout, "virtual"); + break; + case H5D_LAYOUT_ERROR: + case H5D_NLAYOUTS: + error_msg("invalid layout\n"); + return -1; + default: + strcpy(slayout, "invalid layout\n"); + return -1; + } + printf(" Apply %s layout to all\n", slayout); + if (H5D_CHUNKED == options->layout_g) { + printf("with dimension ["); + for (j = 0; j < options->chunk_g.rank; j++) + printf("%d ", (int) options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } + } + }/* verbose */ + + for (i = 0; i < options->op_tbl->nelems; i++) { + char* name = options->op_tbl->objs[i].path; + + if (options->op_tbl->objs[i].chunk.rank > 0) { + if (options->verbose) { + printf(" <%s> with chunk size ", name); + for (k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) + printf("%d ", + (int) options->op_tbl->objs[i].chunk.chunk_lengths[k]); + printf("\n"); + } + has_ck = 1; + } + else if (options->op_tbl->objs[i].chunk.rank == -2) { + if (options->verbose) + printf(" <%s> %s\n", name, "NONE (contigous)"); + has_ck = 1; + } + } + + if (options->all_layout == 1 && has_ck) { + error_msg( + "invalid chunking input: 'all' option\ + is present with other objects\n"); + return -1; + } + + /*------------------------------------------------------------------------- + * objects to filter + *------------------------------------------------------------------------- + */ + + if (options->verbose && have_request(options) /* only print if requested */) { + printf("Objects to apply filter are...\n"); + if (options->all_filter == 1) { + for (k = 0; k < options->n_filter_g; k++) { + H5Z_filter_t filtn = options->filter_g[k].filtn; + switch (filtn) { + case H5Z_FILTER_NONE: + printf(" Uncompress all\n"); + break; + case H5Z_FILTER_SHUFFLE: + case H5Z_FILTER_FLETCHER32: + printf(" All with %s\n", get_sfilter(filtn)); + break; + case H5Z_FILTER_SZIP: + case H5Z_FILTER_DEFLATE: + printf(" All with %s, parameter %d\n", get_sfilter(filtn), + options->filter_g[k].cd_values[0]); + break; + default: + printf(" User Defined %d\n", filtn); + break; + } /* k */ + }; + } + } /* verbose */ + + for (i = 0; i < options->op_tbl->nelems; i++) { + pack_info_t pack = options->op_tbl->objs[i]; + char* name = pack.path; + + for (j = 0; j < pack.nfilters; j++) { + if (options->verbose) { + printf(" <%s> with %s filter\n", name, + get_sfilter(pack.filter[j].filtn)); + } + + has_cp = 1; + + } /* j */ + } /* i */ + + if (options->all_filter == 1 && has_cp) { + error_msg( + "invalid compression input: 'all' option\ + is present with other objects\n"); + return -1; + } + + /*------------------------------------------------------------------------- + * check options for the latest format + *------------------------------------------------------------------------- + */ + + if (options->grp_compact < 0) { + error_msg( + "invalid maximum number of links to store as header messages\n"); + return -1; + } + if (options->grp_indexed < 0) { + error_msg( + "invalid minimum number of links to store in the indexed format\n"); + return -1; + } + if (options->grp_indexed > options->grp_compact) { + error_msg( + "minimum indexed size is greater than the maximum compact size\n"); + return -1; + } + for (i = 0; i < 8; i++) { + if (options->msg_size[i] < 0) { + error_msg("invalid shared message size\n"); + return -1; + } + } + + /*-------------------------------------------------------------------------------- + * verify new user userblock options; file name must be present + *--------------------------------------------------------------------------------- + */ + if (options->ublock_filename != NULL && options->ublock_size == 0) { + if (options->verbose) { + printf( + "Warning: user block size missing for file %s. Assigning a default size of 1024...\n", + options->ublock_filename); + options->ublock_size = 1024; + } + } + + if (options->ublock_filename == NULL && options->ublock_size != 0) { + error_msg("file name missing for user block\n", + options->ublock_filename); + return -1; + } + + /*-------------------------------------------------------------------------------- + * verify alignment options; threshold is zero default but alignment not + *--------------------------------------------------------------------------------- + */ + + if (options->alignment == 0 && options->threshold != 0) { + error_msg("alignment for H5Pset_alignment missing\n"); + return -1; + } + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: check_objects + * + * Purpose: locate all HDF5 objects in the file and compare with user + * supplied list + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ +static int check_objects(const char* fname, pack_opt_t *options) { + hid_t fid; + unsigned int i; + trav_table_t *travt = NULL; + + /* nothing to do */ + if (options->op_tbl->nelems == 0) + return 0; + + /*------------------------------------------------------------------------- + * open the file + *------------------------------------------------------------------------- + */ + if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0)) + < 0) { + printf("<%s>: %s\n", fname, H5FOPENERROR); + return -1; + } + + /*------------------------------------------------------------------------- + * get the list of objects in the file + *------------------------------------------------------------------------- + */ + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fid, travt) < 0) + goto out; + + /*------------------------------------------------------------------------- + * compare with user supplied list + *------------------------------------------------------------------------- + */ + + if (options->verbose) + printf("Opening file <%s>. Searching for objects to modify...\n", + fname); + + for (i = 0; i < options->op_tbl->nelems; i++) { + char* name = options->op_tbl->objs[i].path; + if (options->verbose) + printf(" <%s>", name); + + /* the input object names are present in the file and are valid */ + if (h5trav_getindext(name, travt) < 0) { + error_msg("%s Could not find <%s> in file <%s>. Exiting...\n", + (options->verbose ? "\n" : ""), name, fname); + goto out; + } + if (options->verbose) + printf("...Found\n"); + + /* check for extra filter conditions */ + switch (options->op_tbl->objs[i].filter->filtn) { + /* chunk size must be smaller than pixels per block */ + case H5Z_FILTER_SZIP: + { + int j; + hsize_t csize = 1; + unsigned ppb = options->op_tbl->objs[i].filter->cd_values[0]; + hsize_t dims[H5S_MAX_RANK]; + int rank; + hid_t did; + hid_t sid; + + if (options->op_tbl->objs[i].chunk.rank > 0) { + rank = options->op_tbl->objs[i].chunk.rank; + for (j = 0; j < rank; j++) + csize *= options->op_tbl->objs[i].chunk.chunk_lengths[j]; + } + else { + if ((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + goto out; + if ((sid = H5Dget_space(did)) < 0) + goto out; + if ((rank = H5Sget_simple_extent_ndims(sid)) < 0) + goto out; + HDmemset(dims, 0, sizeof dims); + if (H5Sget_simple_extent_dims(sid, dims, NULL) < 0) + goto out; + for (j = 0; j < rank; j++) + csize *= dims[j]; + if (H5Sclose(sid) < 0) + goto out; + if (H5Dclose(did) < 0) + goto out; + } + + if (csize < ppb) { + printf( + " <warning: SZIP settins, chunk size is smaller than pixels per block>\n"); + goto out; + } + } + break; + default: + break; + } + } /* i */ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + H5Fclose(fid); + trav_table_free(travt); + return 0; + +out: + H5Fclose(fid); + trav_table_free(travt); + return -1; +} + +/*------------------------------------------------------------------------- + * Function: have_request + * + * Purpose: check if a filter or layout was requested + * + * Return: 1 yes, 0 no + * + * Date: May, 24, 2007 + * + *------------------------------------------------------------------------- + */ +static int have_request(pack_opt_t *options) { + + if (options->all_filter || options->all_layout || options->op_tbl->nelems) + return 1; + + return 0; + +} + +/*------------------------------------------------------------------------- + * Function: get_sfilter + * + * Purpose: return the filter as a string name + * + * Return: name of filter, exit on error + * + *------------------------------------------------------------------------- + */ + +static const char* get_sfilter(H5Z_filter_t filtn) { + if (filtn == H5Z_FILTER_NONE) + return "NONE"; + else if (filtn == H5Z_FILTER_DEFLATE) + return "GZIP"; + else if (filtn == H5Z_FILTER_SZIP) + return "SZIP"; + else if (filtn == H5Z_FILTER_SHUFFLE) + return "SHUFFLE"; + else if (filtn == H5Z_FILTER_FLETCHER32) + return "FLETCHER32"; + else if (filtn == H5Z_FILTER_NBIT) + return "NBIT"; + else if (filtn == H5Z_FILTER_SCALEOFFSET) + return "SOFF"; + else + return "UD"; +} + diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h new file mode 100644 index 0000000..d2ab923 --- /dev/null +++ b/tools/src/h5repack/h5repack.h @@ -0,0 +1,243 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#ifndef H5REPACK_H__ +#define H5REPACK_H__ + +#include "H5private.h" +#include "hdf5.h" +#include "h5trav.h" + +#define H5FOPENERROR "unable to open file" +#define PFORMAT "%-7s %-7s %-7s\n" /* chunk info, compression info, name*/ +#define PFORMAT1 "%-7s %-7s %-7s" /* chunk info, compression info, name*/ +#define MAX_NC_NAME 256 /* max length of a name */ +#define MAX_VAR_DIMS 32 /* max per variable dimensions */ +#define FORMAT_OBJ " %-27s %s\n" /* obj type, name */ +#define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ +#define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ + +/*------------------------------------------------------------------------- + * data structures for command line options + *------------------------------------------------------------------------- + */ + +/* a list of names */ +typedef struct { + char obj[MAX_NC_NAME]; +} obj_list_t; + +/* + the type of filter and additional parameter + type can be one of the filters + H5Z_FILTER_NONE 0, uncompress if compressed + H5Z_FILTER_DEFLATE 1 , deflation like gzip + H5Z_FILTER_SHUFFLE 2 , shuffle the data + H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC + H5Z_FILTER_SZIP 4 , szip compression + H5Z_FILTER_NBIT 5 , nbit compression + H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression +*/ + +#define CD_VALUES 20 + +typedef struct { + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[CD_VALUES]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ +} filter_info_t; + +/* chunk lengths along each dimension and rank */ +typedef struct { + hsize_t chunk_lengths[MAX_VAR_DIMS]; + int rank; +} chunk_info_t; + +/* we currently define a maximum value for the filters array, + that corresponds to the current library filters */ +#define H5_REPACK_MAX_NFILTERS 6 + +/* information for one object, contains PATH, CHUNK info and FILTER info */ +typedef struct { + char path[MAX_NC_NAME]; /* name of object */ + filter_info_t filter[H5_REPACK_MAX_NFILTERS]; /* filter array */ + int nfilters; /* current number of filters */ + H5D_layout_t layout; /* layout information */ + chunk_info_t chunk; /* chunk information */ + hid_t refobj_id; /* object ID, references */ +} pack_info_t; + +/* store a table of all objects */ +typedef struct { + unsigned int size; + unsigned int nelems; + pack_info_t *objs; +} pack_opttbl_t; + + +/*------------------------------------------------------------------------- + * command line options + *------------------------------------------------------------------------- + */ + +/* all the above, ready to go to the hrepack call */ +typedef struct { + pack_opttbl_t *op_tbl; /*table with all -c and -f options */ + int all_layout; /*apply the layout to all objects */ + int all_filter; /*apply the filter to all objects */ + filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ + int n_filter_g; /*number of global filters */ + chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /*global layout information for the ALL case */ + int verbose; /*verbose mode */ + hsize_t min_comp; /*minimum size to compress, in bytes */ + int use_native; /*use a native type in write */ + hbool_t latest; /*pack file with the latest file format */ + int grp_compact; /* Set the maximum number of links to store as header messages in the group */ + int grp_indexed; /* Set the minimum number of links to store in the indexed format */ + int msg_size[8]; /* Minimum size of shared messages: dataspace, + datatype, fill value, filter pipleline, attribute */ + const char *ublock_filename; /* user block file name */ + hsize_t ublock_size; /* user block size */ + hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ + hsize_t threshold; /* alignment threshold for H5Pset_alignment */ + hsize_t alignment; /* alignment for H5Pset_alignment */ + H5F_file_space_type_t fs_strategy; /* File space handling strategy */ + hsize_t fs_threshold; /* Free space section threshold */ +} pack_opt_t; + + +typedef struct named_dt_t { + haddr_t addr_in; /* Address of the named dtype in the in file */ + hid_t id_out; /* Open identifier for the dtype in the out file */ + struct named_dt_t *next; /* Next dtype */ +} named_dt_t; + +/*------------------------------------------------------------------------- + * public functions + *------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +extern "C" { +#endif + +int h5repack(const char* infile, const char* outfile, pack_opt_t *options); +int h5repack_addfilter(const char* str, pack_opt_t *options); +int h5repack_addlayout(const char* str, pack_opt_t *options); +int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, + H5F_file_space_type_t strategy, hsize_t threshold); +int h5repack_end(pack_opt_t *options); +int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); +int h5repack_cmp_pl(const char *fname1, const char *fname2); + +/* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() + * and struct named_dt_t were located in h5repack_copy.c as static prior to + * bugfix1726. + * Made shared functions as copy_attr() was needed in h5repack_refs.c. + * However copy_attr() may be obsoleted when H5Acopy is available and put back + * others to static in h5repack_copy.c. + */ +hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options); +int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err); +int copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, + trav_table_t *travt, pack_opt_t *options); + +#ifdef __cplusplus +} +#endif + + + +/*------------------------------------------------------------------------- + * private functions + *------------------------------------------------------------------------- + */ + + +/*------------------------------------------------------------------------- + * copy module + *------------------------------------------------------------------------- + */ + +int copy_objects (const char* fnamein, + const char* fnameout, + pack_opt_t *options); + +int do_copy_refobjs(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options); + +/*------------------------------------------------------------------------- + * filters and verify module + *------------------------------------------------------------------------- + */ +void init_packobject(pack_info_t *obj); + + +/*------------------------------------------------------------------------- + * filters and copy module + *------------------------------------------------------------------------- + */ + +int apply_filters(const char* name, /* object name from traverse list */ + int rank, /* rank of dataset */ + hsize_t *dims, /* dimensions of dataset */ + size_t msize, /* size of type */ + hid_t dcpl_id, /* dataset creation property list */ + pack_opt_t *options, /* repack options */ + int *has_filter); /* (OUT) object NAME has a filter */ + + +/*------------------------------------------------------------------------- + * options table + *------------------------------------------------------------------------- + */ +int options_table_init( pack_opttbl_t **tbl ); +int options_table_free( pack_opttbl_t *table ); +int options_add_layout( obj_list_t *obj_list, + unsigned n_objs, + pack_info_t *pack, + pack_opttbl_t *table ); +int options_add_filter ( obj_list_t *obj_list, + unsigned n_objs, + filter_info_t filt, + pack_opttbl_t *table ); +pack_info_t* options_get_object( const char *path, + pack_opttbl_t *table); + +/*------------------------------------------------------------------------- + * parse functions + *------------------------------------------------------------------------- + */ + +obj_list_t* parse_filter(const char *str, + unsigned *n_objs, + filter_info_t *filt, + pack_opt_t *options, + int *is_glb); + +obj_list_t* parse_layout(const char *str, + unsigned *n_objs, + pack_info_t *pack, /* info about object */ + pack_opt_t *options); + + + + +#endif /* H5REPACK_H__ */ + diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c new file mode 100644 index 0000000..547f61a --- /dev/null +++ b/tools/src/h5repack/h5repack_copy.c @@ -0,0 +1,1581 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/*------------------------------------------------------------------------- + * typedefs + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * globals + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * macros + *------------------------------------------------------------------------- + */ + +/* size of buffer/# of bytes to xfer at a time when copying userblock */ +#define USERBLOCK_XFER_SIZE 512 + +/* check H5Dread()/H5Dwrite() error, e.g. memory allocation error inside the library. */ +#define CHECK_H5DRW_ERROR(_fun, _fail, _did, _mtid, _msid, _fsid, _pid, _buf) { \ + H5E_BEGIN_TRY { \ + if(_fun(_did, _mtid, _msid, _fsid, _pid, _buf) < 0) { \ + hid_t _err_num = 0; \ + char _msg[80]; \ + H5Ewalk2(H5E_DEFAULT, H5E_WALK_DOWNWARD, walk_error_callback, &_err_num); \ + H5Eget_msg(_err_num, NULL, _msg, (size_t)80); \ + error_msg("%s %s -- %s\n", #_fun, "failed", _msg); \ + HGOTO_DONE(_fail) \ + } \ + } H5E_END_TRY; \ +} + +/*------------------------------------------------------------------------- + * local functions + *------------------------------------------------------------------------- + */ +static int Get_hyperslab(hid_t dcpl_id, int rank_dset, hsize_t dims_dset[], + size_t size_datum, hsize_t dims_hslab[], hsize_t * hslab_nbytes_p); +static void print_dataset_info(hid_t dcpl_id, char *objname, double per, int pr); +static int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, + pack_opt_t *options); +static int copy_user_block(const char *infile, const char *outfile, + hsize_t size); +#if defined (H5REPACK_DEBUG_USER_BLOCK) +static void print_user_block(const char *filename, hid_t fid); +#endif +static herr_t walk_error_callback(unsigned n, const H5E_error2_t *err_desc, void *udata); + +/* get the major number from the error stack. */ +static herr_t walk_error_callback(H5_ATTR_UNUSED unsigned n, const H5E_error2_t *err_desc, void *udata) { + if (err_desc) + *((hid_t *) udata) = err_desc->maj_num; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: copy_objects + * + * Purpose: duplicate all HDF5 objects in the file + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 23, 2003 + * + * Modification: + * Peter Cao, June 13, 2007 + * Add "-L, --latest" and other options to pack a file with the latest file format + * + * Peter Cao, September 25, 2007 + * Copy user block when repacking a file + * + * Pedro Vicente, August 20, 2008 + * Add a user block to file if requested + * + *------------------------------------------------------------------------- + */ + +int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t fidin; + hid_t fidout = -1; + trav_table_t *travt = NULL; + hsize_t ub_size = 0; /* size of user block */ + hid_t fcpl = H5P_DEFAULT; /* file creation property list ID */ + hid_t fapl = H5P_DEFAULT; /* file access property list ID */ + + /*------------------------------------------------------------------------- + * open input file + *------------------------------------------------------------------------- + */ + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, (size_t) 0)) < 0) { + error_msg("<%s>: %s\n", fnamein, H5FOPENERROR); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /* get user block size and file space strategy/threshold */ + { + hid_t fcpl_in; /* file creation property list ID for input file */ + + if ((fcpl_in = H5Fget_create_plist(fidin)) < 0) { + error_msg("failed to retrieve file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + if (H5Pget_userblock(fcpl_in, &ub_size) < 0) { + error_msg("failed to retrieve userblock size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + if (!options->fs_strategy) { + if (H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) { + error_msg("failed to retrieve file space strategy\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + if (!options->fs_threshold) { + if (H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) { + error_msg("failed to retrieve file space threshold\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + if (H5Pclose(fcpl_in) < 0) { + error_msg("failed to close property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /* Check if we need to create a non-default file creation property list */ + if (options->latest || ub_size > 0) { + /* Create file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + error_msg("fail to create a file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + if (ub_size > 0) { + if (H5Pset_userblock(fcpl, ub_size) < 0) { + error_msg("failed to set non-default userblock size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + if (options->latest) { + unsigned i = 0, nindex = 0, mesg_type_flags[5], min_mesg_sizes[5]; + + /* Adjust group creation parameters for root group */ + /* (So that it is created in "dense storage" form) */ + if (H5Pset_link_phase_change(fcpl, (unsigned) options->grp_compact, + (unsigned) options->grp_indexed) < 0) { + error_msg("fail to adjust group creation parameters for root group\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + for (i = 0; i < 5; i++) { + if (options->msg_size[i] > 0) { + switch (i) { + case 0: + mesg_type_flags[nindex] = H5O_SHMESG_SDSPACE_FLAG; + break; + + case 1: + mesg_type_flags[nindex] = H5O_SHMESG_DTYPE_FLAG; + break; + + case 2: + mesg_type_flags[nindex] = H5O_SHMESG_FILL_FLAG; + break; + + case 3: + mesg_type_flags[nindex] = H5O_SHMESG_PLINE_FLAG; + break; + + case 4: + mesg_type_flags[nindex] = H5O_SHMESG_ATTR_FLAG; + break; + + default: + break; + } /* end switch */ + min_mesg_sizes[nindex] = (unsigned) options->msg_size[i]; + + nindex++; + } /* end if */ + } /* end for */ + + if (nindex > 0) { + if (H5Pset_shared_mesg_nindexes(fcpl, nindex) < 0) { + error_msg("fail to set the number of shared object header message indexes\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /* msg_size[0]=dataspace, 1=datatype, 2=file value, 3=filter pipleline, 4=attribute */ + for (i = 0; i < (nindex - 1); i++) { + if (H5Pset_shared_mesg_index(fcpl, i, mesg_type_flags[i], min_mesg_sizes[i]) < 0) { + error_msg("fail to configure the specified shared object header message index\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } /* end if */ + } /* end for */ + } /* if (nindex>0) */ + + /* Create file access property list */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + error_msg("Could not create file access property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } /* end if */ + + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) { + error_msg("Could not set property for using latest version of the format\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } /* end if */ + } /* end if */ + } /* end if */ +#if defined (H5REPACK_DEBUG_USER_BLOCK) +print_user_block(fnamein, fidin); +#endif + + /*------------------------------------------------------------------------- + * set the new user userblock options in the FCPL (before H5Fcreate ) + *------------------------------------------------------------------------- + */ + if (options->ublock_size > 0) { + /* either use the FCPL already created or create a new one */ + if (fcpl == H5P_DEFAULT) { + /* create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + error_msg("fail to create a file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /* set user block size */ + if (H5Pset_userblock(fcpl, options->ublock_size) < 0) { + error_msg("failed to set userblock size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /*------------------------------------------------------------------------- + * set alignment options + *------------------------------------------------------------------------- + */ + if (options->alignment > 0) { + /* either use the FAPL already created or create a new one */ + if (fapl == H5P_DEFAULT) { + /* create a file access property list */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + error_msg("Could not create file access property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0) { + error_msg("failed to set alignment\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /*------------------------------------------------------------------------- + * set metadata block size option + *------------------------------------------------------------------------- + */ + if (options->meta_block_size > 0) { + /* either use the FAPL already created or create a new one */ + if (fapl == H5P_DEFAULT) { + /* create a file access property list */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + error_msg("Could not create file access property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + if (H5Pset_meta_block_size(fapl, options->meta_block_size) < 0) { + error_msg("failed to set metadata block size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /*------------------------------------------------------------------------- + * set free-space strategy options + *------------------------------------------------------------------------- + */ + + /* either use the FCPL already created or create a new one */ + if (fcpl == H5P_DEFAULT) { + /* create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + error_msg("fail to create a file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /* set file space strategy and free space threshold */ + if (H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) { + error_msg("failed to set file space strategy & threshold \n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /*------------------------------------------------------------------------- + * create the output file + *------------------------------------------------------------------------- + */ + if (options->verbose) + printf("Making file <%s>...\n", fnameout); + + if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, fapl)) < 0) { + error_msg("<%s>: Could not create file\n", fnameout); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /*------------------------------------------------------------------------- + * write a new user block if requested + *------------------------------------------------------------------------- + */ + if (options->ublock_size > 0) { + if (copy_user_block(options->ublock_filename, fnameout, options->ublock_size) < 0) { + error_msg("Could not copy user block. Exiting...\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /*------------------------------------------------------------------------- + * get list of objects + *------------------------------------------------------------------------- + */ + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fidin, travt) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + + /*------------------------------------------------------------------------- + * do the copy + *------------------------------------------------------------------------- + */ + if (do_copy_objects(fidin, fidout, travt, options) < 0) { + error_msg("<%s>: Could not copy data to: %s\n", fnamein, fnameout); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } /* end if */ + + /*------------------------------------------------------------------------- + * do the copy of referenced objects + * and create hard links + *------------------------------------------------------------------------- + */ + if (do_copy_refobjs(fidin, fidout, travt, options) < 0) { + printf("h5repack: <%s>: Could not copy data to: %s\n", fnamein, fnameout); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (fapl > 0) + H5Pclose(fapl); + + if (fcpl > 0) + H5Pclose(fcpl); + + H5Fclose(fidin); + H5Fclose(fidout); + + /* free table */ + trav_table_free(travt); + travt = NULL; + + /*------------------------------------------------------------------------- + * write only the input file user block if there is no user block file input + *------------------------------------------------------------------------- + */ + + if (ub_size > 0 && options->ublock_size == 0) { + if (copy_user_block(fnamein, fnameout, ub_size) < 0) { + error_msg("Could not copy user block. Exiting...\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + return 0; + + /*------------------------------------------------------------------------- + * out + *------------------------------------------------------------------------- + */ + +done: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fidin); + H5Fclose(fidout); + } H5E_END_TRY; + if (travt) + trav_table_free(travt); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: Get_hyperslab + * + * Purpose: Calulate a hyperslab from a dataset for higher performance. + * The size of hyperslab is limitted by H5TOOLS_BUFSIZE. + * Return the hyperslab dimentions and size in byte. + * + * Return: 0 - SUCCEED, -1 FAILED + * + * Parameters: + * dcpl_id : [IN] dataset creation property. + * rank_dset : [IN] dataset rank + * dims_dset[] : [IN] dataset dimentions + * size_datum : [IN] size of a data element in byte + * dims_hslab[] : [OUT] calculated hyperslab dimentions + * * hslab_nbytes_p : [OUT] total byte of the hyperslab + * + * Programmer: Jonathan Kim + * Date: Feburary, 2012 + * Update: + * The hyperslab calucation would be depend on if the dataset is chunked + * or not. + * + * There care 3 conditions to cover: + * 1. If chunked and a chunk fits in buffer, each chunk would be a unit of + * collection and the boundary would be dataset's dims. + * 2. If chunked but a chunk doesn't fit in buffer, each data element would + * be a unit of collection and the boundary would be the chunk itself. + * 3. If not chunked, each data element would be a unit of collection and + * the boundary would be dataset's dims. + * + * The calulation starts from the last dimention (h5dump dims output). + * + * Note: + * Added for JIRA HDFFV-7862. + *-----------------------------------------*/ + +int Get_hyperslab(hid_t dcpl_id, int rank_dset, hsize_t dims_dset[], + size_t size_datum, hsize_t dims_hslab[], hsize_t * hslab_nbytes_p) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + int k; + H5D_layout_t dset_layout; + int rank_chunk; + hsize_t dims_chunk[H5S_MAX_RANK]; + hsize_t size_chunk = 1; + hsize_t nchunk_fit; /* number of chunks that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ + hsize_t ndatum_fit; /* number of dataum that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ + hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimentions */ + hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimentions */ + hsize_t hslab_nbytes; /* size of hyperslab in byte */ + + /* init to set as size of a data element */ + hslab_nbytes = size_datum; + + /* get layout of dataset */ + dset_layout = H5Pget_layout(dcpl_id); + + /* if dataset is chunked */ + if (dset_layout == H5D_CHUNKED) { + /* get chunk dims */ + rank_chunk = H5Pget_chunk(dcpl_id, rank_dset, dims_chunk); + if (rank_chunk < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + + for (k = rank_dset; k > 0; --k) + size_chunk *= dims_chunk[k - 1]; + + /* figure out how many chunks can fit in the hyperslab buffer */ + nchunk_fit = (H5TOOLS_BUFSIZE / size_datum) / size_chunk; + + /* 1. if a chunk fit in hyperslab buffer */ + if (nchunk_fit >= 1) { + /* Calulate a hyperslab that contains as many chunks that can fit + * in hyperslab buffer. Hyperslab will be increased starting from + * the last dimention of the dataset (see h5dump's dims output). + * The calculation boundary is dataset dims. + * In the loop, used mapping from a datum to a chunk to figure out + * chunk based hyperslab. + */ + for (k = rank_dset; k > 0; --k) { + /* map dataset dimentions with a chunk dims */ + chunk_dims_map[k - 1] = dims_dset[k - 1] / dims_chunk[k - 1]; + + /* if reminder exist, increse by 1 to cover partial edge chunks */ + if (dims_dset[k - 1] % dims_chunk[k - 1] > 0) + chunk_dims_map[k - 1]++; + + /* get mapped hyperslab dims */ + hs_dims_map[k - 1] = MIN (nchunk_fit, chunk_dims_map[k-1]); + + /* prepare next round */ + nchunk_fit = nchunk_fit / chunk_dims_map[k - 1]; + /* if a chunk is bigger than the rest of buffer */ + if (nchunk_fit == 0) + nchunk_fit = 1; + + /* get hyperslab dimentions as unmapping to actual size */ + dims_hslab[k - 1] = MIN( (hs_dims_map[k-1] * dims_chunk[k-1]), dims_dset[k-1]); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k - 1]; + } + } + /* 2. if a chunk is bigger than hyperslab buffer */ + else { + /* Calulate a hyperslab that contains as many data elements that + * can fit in hyperslab buffer. Hyperslab will be increased + * starting from the last dimention of the chunk (see h5dump's dims + * output). + * The calculation boundary is a chunk dims. + */ + for (k = rank_dset; k > 0; --k) { + ndatum_fit = H5TOOLS_BUFSIZE / hslab_nbytes; + + /* if a datum is bigger than rest of buffer */ + if (ndatum_fit == 0) + ndatum_fit = 1; + /* get hyperslab dimentions within a chunk boundary */ + dims_hslab[k - 1] = MIN (dims_chunk[k-1], ndatum_fit); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k - 1]; + + if (hslab_nbytes <= 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + } + /* 3. if dataset is not chunked */ + else { + /* Calulate a hyperslab that contains as many data elements that can + * fit in hyperslab buffer. Hyperslab will be increased starting from + * the last dimention of the dataset (see h5dump's dims output). + * The calculation boundary is dataset dims. + */ + for (k = rank_dset; k > 0; --k) { + ndatum_fit = H5TOOLS_BUFSIZE / hslab_nbytes; + + /* if a datum is bigger than rest of buffer */ + if (ndatum_fit == 0) + ndatum_fit = 1; + /* get hyperslab dimentions within dataset boundary */ + dims_hslab[k - 1] = MIN(dims_dset[k - 1], ndatum_fit); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k - 1]; + + if (hslab_nbytes <= 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + } + + /* pass out the hyperslab size*/ + *hslab_nbytes_p = hslab_nbytes; + +done: + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: do_copy_objects + * + * Purpose: duplicate all HDF5 objects in the file + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 23, 2003 + * + * Modifications: + * + * July 2004: Introduced the extra EC or NN option for SZIP + * + * December 2004: Added a check for H5Dcreate; if the dataset cannot be created + * with the requested filter, use the input one + * + * October 2006: Read/write using the file type by default. + * + * October 2006: Read by hyperslabs for big datasets. + * + * A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done + * i.e., if the memory needed to read a dataset is greater than this limit, + * then hyperslab I/O is done instead of one operation I/O + * For each dataset, the memory needed is calculated according to + * + * memory needed = number of elements * size of each element + * + * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations + * are done + * + * H5Dread( input_dataset1 ) + * H5Dread( input_dataset2 ) + * + * with all elements in the datasets selected. If the memory needed is greater than + * H5TOOLS_MALLOCSIZE, then the following operations are done instead: + * + * a strip mine is defined for each dimension k (a strip mine is defined as a + * hyperslab whose size is memory manageable) according to the formula + * + * (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type) + * + * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures + * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip + * mine size k is simply defined as its dimension k, but for larger datasets the + * hyperslab size is still memory manageable. + * a cycle is done until the number of elements in the dataset is reached. In each + * iteration, two parameters are defined for the function H5Sselect_hyperslab, + * the start and size of each hyperslab, according to + * + * (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k]) + * + * where hyperslab_offset [k] is initially set to zero, and later incremented in + * hyperslab_size[k] offsets. The reason for the operation + * + * dimension[k] - hyperslab_offset[k] + * + * in (2) is that, when using the strip mine size, it assures that the "remaining" part + * of the dataset that does not fill an entire strip mine is processed. + * + * November 2006: Use H5Ocopy in the copy of objects. The logic for using + * H5Ocopy or not is if a change of filters or layout is requested by the user + * then use read/write else use H5Ocopy. + * + * May, 1, 2008: Add a printing of the compression ratio of old size / new size + * + * Feburary 2012: improve Read/Write by hyperslabs for big datasets. + * Programmer: Jonathan Kim + * + * A threshold of H5TOOLS_MALLOCSIZE is the limit upon which I/O hyperslab is done + * i.e., if the memory needed to read a dataset is greater than this limit, + * then hyperslab I/O is done instead of one operation I/O + * For each dataset, the memory needed is calculated according to + * + * memory needed = number of elements * size of each element + * + * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations + * are done + * + * H5Dread( input_dataset ) + * H5Dwrite( output_dataset ) + * + * with all elements in the datasets selected. If the memory needed is greater than + * H5TOOLS_MALLOCSIZE, then the following operations are done instead: + * + * 1. figure out a hyperslab (dimentions) and size (refer to Get_hyperslab()). + * 2. Calculate the hyperslab selections as the selection is moving forward. + * Selection would be same as the hyperslab except for the remaining edge portion + * of the dataset. The code take care of the remaining portion if exist. + * + *------------------------------------------------------------------------- + */ + +int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, + pack_opt_t *options) /* repack options */ +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t grp_in = -1; /* group ID */ + hid_t grp_out = -1; /* group ID */ + hid_t dset_in = -1; /* read dataset ID */ + hid_t dset_out = -1; /* write dataset ID */ + hid_t gcpl_in = -1; /* group creation property list */ + hid_t gcpl_out = -1; /* group creation property list */ + hid_t type_in = -1; /* named type ID */ + hid_t type_out = -1; /* named type ID */ + hid_t dcpl_in = -1; /* dataset creation property list ID */ + hid_t dcpl_out = -1; /* dataset creation property list ID */ + hid_t f_space_id = -1; /* file space ID */ + hid_t ftype_id = -1; /* file type ID */ + hid_t wtype_id = -1; /* read/write type ID */ + named_dt_t *named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ + size_t msize; /* size of type */ + hsize_t nelmts; /* number of elements in dataset */ + H5D_space_status_t space_status; /* determines whether space has been allocated for the dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ + hsize_t dsize_out; /* output dataset size after filter */ + int apply_s; /* flag for apply filter to small dataset sizes */ + int apply_f; /* flag for apply filter to return error on H5Dcreate */ + void *buf = NULL; /* buffer for raw data */ + void *hslab_buf = NULL; /* hyperslab buffer for raw data */ + int has_filter; /* current object has a filter */ + int req_filter; /* there was a request for a filter */ + int req_obj_layout = 0; /* request layout to current object */ + unsigned crt_order_flags; /* group creation order flag */ + unsigned i; + unsigned u; + int is_ref = 0; + htri_t is_named; + hbool_t limit_maxdims; + hsize_t size_dset; + + /*------------------------------------------------------------------------- + * copy the suppplied object list + *------------------------------------------------------------------------- + */ + + if (options->verbose) { + printf("-----------------------------------------\n"); + printf(" Type Filter (Compression) Name\n"); + printf("-----------------------------------------\n"); + } + + for (i = 0; i < travt->nobjs; i++) { + /* init variables per obj */ + buf = NULL; + limit_maxdims = FALSE; + + switch (travt->objs[i].type) { + case H5TRAV_TYPE_UNKNOWN: + HDassert(0); + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_GROUP + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_GROUP: + if (options->verbose) + printf(FORMAT_OBJ, "group", travt->objs[i].name); + + /* open input group */ + if ((grp_in = H5Gopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + + /* get input group creation property list */ + if ((gcpl_in = H5Gget_create_plist(grp_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed"); + + /* query and set the group creation properties */ + if (H5Pget_link_creation_order(gcpl_in, &crt_order_flags) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed"); + + /* set up group creation property list */ + if ((gcpl_out = H5Pcreate(H5P_GROUP_CREATE)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed"); + + if (H5Pset_link_creation_order(gcpl_out, crt_order_flags) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_creation_order failed"); + + /*------------------------------------------------------------------------- + * the root is a special case, we get an ID for the root group + * and copy its attributes using that ID + *------------------------------------------------------------------------- + */ + if (HDstrcmp(travt->objs[i].name, "/") == 0) { + if ((grp_out = H5Gopen2(fidout, "/", H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + } + else { + if (options->grp_compact > 0 || options->grp_indexed > 0) + if (H5Pset_link_phase_change(gcpl_out, (unsigned) options->grp_compact, (unsigned) options->grp_indexed) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_phase_change failed"); + + if ((grp_out = H5Gcreate2(fidout, travt->objs[i].name, H5P_DEFAULT, gcpl_out, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gcreate2 failed"); + } + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(grp_in, grp_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + + if (H5Pclose(gcpl_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Pclose(gcpl_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Gclose(grp_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + if (H5Gclose(grp_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_DATASET + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_DATASET: + has_filter = 0; + req_filter = 0; + + /* check if global filters were requested */ + if (options->n_filter_g) + req_filter = 1; + + /* check if filters were requested for individual objects */ + for (u = 0; u < options->op_tbl->nelems; u++) + if (HDstrcmp(travt->objs[i].name, options->op_tbl->objs[u].path) == 0) + if (options->op_tbl->objs[u].filter->filtn > 0) + req_filter = 1; + + /* check if layout change requested individual object */ + if (options->layout_g != H5D_LAYOUT_ERROR) { + pack_info_t *pckinfo; + + /* any dataset is specified */ + if (options->op_tbl->nelems > 0) { + /* check if object exist */ + pckinfo = options_get_object(travt->objs[i].name, options->op_tbl); + if (pckinfo) + req_obj_layout = 1; + } + } + + /* early detection of references */ + if ((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if ((ftype_id = H5Dget_type(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); + if (H5T_REFERENCE == H5Tget_class(ftype_id)) + is_ref = 1; + + /* Check if the datatype is committed */ + if ((is_named = H5Tcommitted(ftype_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tcommitted failed"); + if (is_named) + if ((wtype_id = copy_named_datatype(ftype_id, fidout, &named_dt_head, travt, options)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_named_datatype failed"); + + if (H5Tclose(ftype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Dclose(dset_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + + /*------------------------------------------------------------------------- + * check if we should use H5Ocopy or not + * if there is a request for filters/layout, we read/write the object + * otherwise we do a copy using H5Ocopy + *------------------------------------------------------------------------- + */ + if (options->op_tbl->nelems || options->all_filter == 1 + || options->all_layout == 1 || is_ref || is_named) { + + int j; + + if ((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if ((f_space_id = H5Dget_space(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); + if ((ftype_id = H5Dget_type(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); + if ((dcpl_in = H5Dget_create_plist(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + if ((dcpl_out = H5Pcopy(dcpl_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed"); + if ((rank = H5Sget_simple_extent_ndims(f_space_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed"); + HDmemset(dims, 0, sizeof dims); + if (H5Sget_simple_extent_dims(f_space_id, dims, NULL) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed"); + if (H5Dget_space_status(dset_in, &space_status) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space_status failed"); + + nelmts = 1; + for (j = 0; j < rank; j++) + nelmts *= dims[j]; + + /* wtype_id will have already been set if using a named dtype */ + if (!is_named) { + if (options->use_native == 1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); + } /* end if */ + + if ((msize = H5Tget_size(wtype_id)) == 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); + + /* size of current dset */ + size_dset = nelmts * msize; + + /*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- + */ + if (h5tools_canreadf((travt->objs[i].name), dcpl_in) == 1) { + apply_s = 1; + apply_f = 1; + + /*------------------------------------------------------------------------- + * references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * in a second traversal of the output file + *------------------------------------------------------------------------- + */ + if (H5T_REFERENCE != H5Tget_class(wtype_id)) { + /* get the storage size of the input dataset */ + dsize_in = H5Dget_storage_size(dset_in); + + /* check for small size datasets (less than 1k) except + * changing to COMPACT. For the reference, COMPACT is limited + * by size 64K by library. + */ + if (options->layout_g != H5D_COMPACT) + if (size_dset < options->min_comp) + apply_s = 0; + + /* apply the filter */ + if (apply_s) + if (apply_filters(travt->objs[i].name, rank, dims, msize, dcpl_out, options, &has_filter) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "apply_filters failed"); + + /* only if layout change requested for entire file or + * individual obj */ + if (options->all_layout > 0 || req_obj_layout == 1) + /*------------------------------------------------- + * Unset the unlimited max dims if convert to other + * than chunk layouts, because unlimited max dims + * only can be applied to chunk layout. + * Also perform only for targeted dataset + * Also check for size limit to convert to compact + *-------------------------------------------------*/ + if (options->layout_g != H5D_CHUNKED) { + /* any dataset is specified */ + if (options->op_tbl->nelems > 0) { + /* if current obj match specified obj */ + if (options_get_object(travt->objs[i].name, options->op_tbl)) + limit_maxdims = TRUE; + } + else /* no dataset is specified */ + limit_maxdims = TRUE; + + /* if convert to COMPACT */ + if (options->layout_g == H5D_COMPACT) + /* should be smaller than 64K */ + if (size_dset > MAX_COMPACT_DSIZE) + limit_maxdims = FALSE; + + /* unset unlimited max dims */ + if (limit_maxdims) + H5Sset_extent_simple(f_space_id, rank, dims, NULL); + } + + /*------------------------------------------------------------------------- + * create the output dataset; + * disable error checking in case the dataset cannot be created with the + * modified dcpl; in that case use the original instead + *------------------------------------------------------------------------- + */ + dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT); + if (dset_out == FAIL) { + H5Epush2(H5tools_ERR_STACK_g, __FILE__, FUNC, __LINE__, H5tools_ERR_CLS_g, H5E_tools_g, H5E_tools_min_id_g, "H5Dcreate2 failed"); + if (options->verbose) + printf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name); + + if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed"); + apply_f = 0; + } + + /*------------------------------------------------------------------------- + * read/write + *------------------------------------------------------------------------- + */ + if (nelmts > 0 && space_status != H5D_SPACE_STATUS_NOT_ALLOCATED) { + size_t need = (size_t)(nelmts * msize); /* bytes needed */ + + /* have to read the whole dataset if there is only one element in the dataset */ + if (need < H5TOOLS_MALLOCSIZE) + buf = HDmalloc(need); + + if (buf != NULL) { + /* read/write: use the macro to check error, e.g. memory allocation error inside the library. */ + CHECK_H5DRW_ERROR(H5Dread, FAIL, dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + CHECK_H5DRW_ERROR(H5Dwrite, FAIL, dset_out, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + + /* Check if we have VL data in the dataset's + * datatype that must be reclaimed */ + if (TRUE == H5Tdetect_class(wtype_id, H5T_VLEN)) + if (H5Dvlen_reclaim(wtype_id, f_space_id, H5P_DEFAULT, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dvlen_reclaim failed"); + } + else { /* possibly not enough memory, read/write by hyperslabs */ + size_t p_type_nbytes = msize; /*size of memory type */ + hsize_t p_nelmts = nelmts; /*total elements */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* hyperslab info */ + hsize_t hslab_dims[H5S_MAX_RANK]; /*hyperslab dims */ + hsize_t hslab_nbytes; /*bytes per hyperslab */ + hsize_t hslab_nelmts; /*elements per hyperslab*/ + hid_t hslab_space; /*hyperslab data space */ + + /* hyperslab selection info */ + hsize_t hs_sel_offset[H5S_MAX_RANK];/* selection offset */ + hsize_t hs_sel_count[H5S_MAX_RANK]; /* selection count */ + hsize_t hs_select_nelmts; /* selected elements */ + hsize_t zero[8]; /*vector of zeros */ + int k; + H5D_layout_t dset_layout; + hid_t dcpl_tmp = -1; /* dataset creation property list ID */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* check first if writing dataset is chunked, + * if so use its chunk layout for better performance. */ + dset_layout = H5Pget_layout(dcpl_out); + if (dset_layout == H5D_CHUNKED) + dcpl_tmp = dcpl_out; /* writing dataset */ + else { /* if reading dataset is chunked */ + dset_layout = H5Pget_layout(dcpl_in); + if (dset_layout == H5D_CHUNKED) + dcpl_tmp = dcpl_in; /* reading dataset */ + } + + /* get hyperslab dims and size in byte */ + if (Get_hyperslab(dcpl_tmp, rank, dims, p_type_nbytes, hslab_dims, &hslab_nbytes) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Get_hyperslab failed"); + + hslab_buf = HDmalloc((size_t)hslab_nbytes); + + hslab_nelmts = hslab_nbytes / p_type_nbytes; + hslab_space = H5Screate_simple(1, &hslab_nelmts, NULL); + + /* the hyperslab selection loop */ + HDmemset(hs_sel_offset, 0, sizeof hs_sel_offset); + HDmemset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_select_nelmts) { + if (rank > 0) { + /* calculate the hyperslab selections. + * The selection would be same as the hyperslab + * except for remaining edge portion of the dataset + * which is smaller then the hyperslab. + */ + for (k = 0, hs_select_nelmts = 1; k < rank; k++) { + /* MIN() is used to get the remaining edge portion if exist. + * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is smaller then the hyperslab.*/ + hs_sel_count[k] = MIN(dims[k] - hs_sel_offset[k], hslab_dims[k]); + hs_select_nelmts *= hs_sel_count[k]; + } + + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_sel_offset, NULL, hs_sel_count, NULL) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed"); + if (H5Sselect_hyperslab(hslab_space, H5S_SELECT_SET, zero, NULL, &hs_select_nelmts, NULL) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed"); + } + else { + H5Sselect_all(f_space_id); + H5Sselect_all(hslab_space); + hs_select_nelmts = 1; + } /* rank */ + + /* read/write: use the macro to check error, e.g. memory allocation error inside the library. */ + CHECK_H5DRW_ERROR(H5Dread, FAIL, dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf); + CHECK_H5DRW_ERROR(H5Dwrite, FAIL, dset_out, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf); + + /* reclaim any VL memory, if necessary */ + if (vl_data) + H5Dvlen_reclaim(wtype_id, hslab_space, H5P_DEFAULT, hslab_buf); + + /* calculate the next hyperslab offset */ + for (k = rank, carry = 1; k > 0 && carry; --k) { + hs_sel_offset[k - 1] += hs_sel_count[k - 1]; + /* if reached the end of a dim */ + if (hs_sel_offset[k - 1] == dims[k - 1]) + hs_sel_offset[k - 1] = 0; + else + carry = 0; + } /* k */ + } /* elmtno */ + + H5Sclose(hslab_space); + /* free */ + if (hslab_buf != NULL) { + HDfree(hslab_buf); + hslab_buf = NULL; + } + } /* hyperslab read */ + } /* if (nelmts>0 && space_status==H5D_SPACE_STATUS_NOT_ALLOCATED) */ + + /*------------------------------------------------------------------------- + * amount of compression used + *------------------------------------------------------------------------- + */ + if (options->verbose) { + double ratio = 0; + + /* only print the compression ration if there was a filter request */ + if (apply_s && apply_f && req_filter) { + /* get the storage size of the output dataset */ + dsize_out = H5Dget_storage_size(dset_out); + + /* compression ratio = uncompressed size / compressed size */ + if (dsize_out != 0) + ratio = (double) dsize_in / (double) dsize_out; + print_dataset_info(dcpl_out, travt->objs[i].name, ratio, 1); + } + else + print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0); + + /* print a message that the filter was not applied + (in case there was a filter) + */ + if (has_filter && apply_s == 0) + printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp); + + if (has_filter && apply_f == 0) + printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); + } /* verbose */ + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + + /*close */ + if (H5Dclose(dset_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + }/*!H5T_REFERENCE*/ + }/*h5tools_canreadf*/ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Tclose(ftype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Tclose(wtype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Pclose(dcpl_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Pclose(dcpl_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Sclose(f_space_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if (H5Dclose(dset_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + } + /*------------------------------------------------------------------------- + * we do not have request for filter/chunking use H5Ocopy instead + *------------------------------------------------------------------------- + */ + else { + hid_t pid; + + /* create property to pass copy options */ + if ((pid = H5Pcreate(H5P_OBJECT_COPY)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed"); + + /* set options for object copy */ + if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_copy_object failed"); + + /*------------------------------------------------------------------------- + * do the copy + *------------------------------------------------------------------------- + */ + + if (H5Ocopy(fidin, /* Source file or group identifier */ + travt->objs[i].name, /* Name of the source object to be copied */ + fidout, /* Destination file or group identifier */ + travt->objs[i].name, /* Name of the destination object */ + pid, /* Properties which apply to the copy */ + H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */ + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Ocopy failed"); + + /* close property */ + if (H5Pclose(pid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + + /*------------------------------------------------------------------------- + * copy attrs manually + *------------------------------------------------------------------------- + */ + if ((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if ((dset_out = H5Dopen2(fidout, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if (copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + if (H5Dclose(dset_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + if (H5Dclose(dset_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + + if (options->verbose) + printf(FORMAT_OBJ, "dset", travt->objs[i].name); + + } /* end do we have request for filter/chunking */ + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_NAMED_DATATYPE + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_NAMED_DATATYPE: + if (options->verbose) + printf(FORMAT_OBJ, "type", travt->objs[i].name); + + if ((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Topen2 failed"); + + /* Copy the datatype anonymously */ + if ((type_out = copy_named_datatype(type_in, fidout, &named_dt_head, travt, options)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_named_datatype failed"); + + /* Link in to group structure */ + if (H5Lcreate_hard(type_out, ".", fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Lcreate_hard failed"); + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(type_in, type_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + + if (H5Tclose(type_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if (H5Tclose(type_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_LINK + * H5TRAV_TYPE_UDLINK + * + * Only handles external links; H5Lcopy will fail for other UD link types + * since we don't have creation or copy callbacks for them. + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_LINK: + case H5TRAV_TYPE_UDLINK: + if (options->verbose) + printf(FORMAT_OBJ, "link", travt->objs[i].name); + + if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Lcopy failed"); + + if (options->verbose) + printf(FORMAT_OBJ, "link", travt->objs[i].name); + break; + + default: + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Object type not found"); + } /* switch */ + + /* free */ + if (buf != NULL) { + HDfree(buf); + buf = NULL; + } + } /* i */ + + /* Finalize (link) the stack of named datatypes (if any) */ + named_datatype_free(&named_dt_head, 0); + + return ret_value; + +done: + H5E_BEGIN_TRY + { + H5Gclose(grp_in); + H5Gclose(grp_out); + H5Pclose(dcpl_in); + H5Pclose(gcpl_in); + H5Pclose(gcpl_out); + H5Sclose(f_space_id); + H5Dclose(dset_in); + H5Dclose(dset_out); + H5Tclose(ftype_id); + H5Tclose(wtype_id); + H5Tclose(type_in); + H5Tclose(type_out); + named_datatype_free(&named_dt_head, 1); + }H5E_END_TRY; + + /* free */ + if (buf != NULL) + HDfree(buf); + if (hslab_buf != NULL) + HDfree(hslab_buf); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: print_dataset_info + * + * Purpose: print name, filters, percentage compression of a dataset + * + *------------------------------------------------------------------------- + */ +static void print_dataset_info(hid_t dcpl_id, char *objname, double ratio, + int pr) +{ + char strfilter[255]; +#if defined (PRINT_DEBUG ) + char temp[255]; +#endif + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_objname[256]; /* filter objname */ + int i; + + HDstrcpy(strfilter, "\0"); + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0) + return; + + for (i = 0; i < nfilters; i++) { + cd_nelmts = NELMTS(cd_values); + + filtn = H5Pget_filter2(dcpl_id, (unsigned) i, &filt_flags, &cd_nelmts, + cd_values, sizeof(f_objname), f_objname, NULL); + + switch (filtn) { + case H5Z_FILTER_NONE: + HDstrcat(strfilter, "NONE "); + break; + + case H5Z_FILTER_DEFLATE: + HDstrcat(strfilter, "GZIP "); + +#if defined (PRINT_DEBUG) + { + unsigned level = cd_values[0]; + + sprintf(temp,"(%d)", level); + HDstrcat(strfilter, temp); + } +#endif + break; + + case H5Z_FILTER_SZIP: + HDstrcat(strfilter, "SZIP "); + +#if defined (PRINT_DEBUG) + { + unsigned options_mask = cd_values[0]; /* from dcpl, not filt*/ + unsigned ppb = cd_values[1]; + + sprintf(temp,"(%d,", ppb); + HDstrcat(strfilter, temp); + if (options_mask & H5_SZIP_EC_OPTION_MASK) + HDstrcpy(temp, "EC) "); + else if (options_mask & H5_SZIP_NN_OPTION_MASK) + HDstrcpy(temp, "NN) "); + } + HDstrcat(strfilter, temp); +#endif + break; + + case H5Z_FILTER_SHUFFLE: + HDstrcat(strfilter, "SHUF "); + break; + + case H5Z_FILTER_FLETCHER32: + HDstrcat(strfilter, "FLET "); + break; + + case H5Z_FILTER_NBIT: + HDstrcat(strfilter, "NBIT "); + break; + + case H5Z_FILTER_SCALEOFFSET: + HDstrcat(strfilter, "SCALEOFFSET "); + break; + + default: + HDstrcat(strfilter, "UD "); + break; + } /* switch */ + }/*i*/ + + if (!pr) + printf(FORMAT_OBJ, "dset", objname); + else { + char str[255], temp[28]; + + HDstrcpy(str, "dset "); + HDstrcat(str, strfilter); + sprintf(temp, " (%.3f:1)", ratio); + HDstrcat(str, temp); + printf(FORMAT_OBJ, str, objname); + } +} + +/*------------------------------------------------------------------------- + * Function: copy_user_block + * + * Purpose: copy user block from one file to another + * + * Return: 0, ok, -1 no + * + * Programmer: Peter Cao + * + * Date: October, 25, 2007 + * + *------------------------------------------------------------------------- + */ +static int copy_user_block(const char *infile, const char *outfile, + hsize_t size) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + int infid = -1, outfid = -1; /* File descriptors */ + + /* User block must be any power of 2 equal to 512 or greater (512, 1024, 2048, etc.) */ + HDassert(size > 0); + + /* Open files */ + if ((infid = HDopen(infile, O_RDONLY, 0)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDopen failed"); + if ((outfid = HDopen(outfile, O_WRONLY, 0644)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDopen failed"); + + /* Copy the userblock from the input file to the output file */ + while (size > 0) { + ssize_t nread, nbytes; /* # of bytes transfered, etc. */ + char rbuf[USERBLOCK_XFER_SIZE]; /* Buffer for reading */ + const char *wbuf; /* Pointer into buffer, for writing */ + + /* Read buffer from source file */ + if (size > USERBLOCK_XFER_SIZE) + nread = HDread(infid, rbuf, (size_t)USERBLOCK_XFER_SIZE); + else + nread = HDread(infid, rbuf, (size_t)size); + if (nread < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDread failed"); + + /* Write buffer to destination file */ + /* (compensating for interrupted writes & checking for errors, etc.) */ + nbytes = nread; + wbuf = rbuf; + while (nbytes > 0) { + ssize_t nwritten; /* # of bytes written */ + + do { + nwritten = HDwrite(outfid, wbuf, (size_t)nbytes); + } while (-1 == nwritten && EINTR == errno); + if (-1 == nwritten) /* error */ + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDwrite failed"); + HDassert(nwritten > 0); + HDassert(nwritten <= nbytes); + + /* Update # of bytes left & offset in buffer */ + nbytes -= nwritten; + wbuf += nwritten; + HDassert(nbytes == 0 || wbuf < (rbuf + USERBLOCK_XFER_SIZE)); + } /* end while */ + + /* Update size of userblock left to transfer */ + size = size - (hsize_t) nread; + } /* end while */ + +done: + if (infid > 0) + HDclose(infid); + if (outfid > 0) + HDclose(outfid); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: print_user_block + * + * Purpose: print user block + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente + * + * Date: August, 20, 2008 + * + *------------------------------------------------------------------------- + */ +#if defined (H5REPACK_DEBUG_USER_BLOCK) +static +void print_user_block(const char *filename, hid_t fid) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + int fh; /* file handle */ + hsize_t ub_size; /* user block size */ + hsize_t size; /* size read */ + hid_t fcpl; /* file creation property list ID for HDF5 file */ + int i; + + /* get user block size */ + if(( fcpl = H5Fget_create_plist(fid)) < 0) { + error_msg("failed to retrieve file creation property list\n"); + HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + + if(H5Pget_userblock(fcpl, &ub_size) < 0) { + error_msg("failed to retrieve userblock size\n"); + HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "H5Pget_userblock failed"); + } + + if(H5Pclose(fcpl) < 0) { + error_msg("failed to close property list\n"); + HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "H5Pclose failed"); + } + + /* open file */ + if((fh = HDopen(filename, O_RDONLY, 0)) < 0) { + HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "HDopen failed"); + } + + size = ub_size; + + /* read file */ + while(size > 0) { + ssize_t nread; /* # of bytes read */ + char rbuf[USERBLOCK_XFER_SIZE]; /* buffer for reading */ + + /* read buffer */ + if(size > USERBLOCK_XFER_SIZE) + nread = HDread(fh, rbuf, (size_t)USERBLOCK_XFER_SIZE); + else + nread = HDread(fh, rbuf, (size_t)size); + + for(i = 0; i < nread; i++) { + + printf("%c ", rbuf[i]); + + } + printf("\n"); + + if(nread < 0) { + HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "nread < 0"); + } + + /* update size of userblock left to transfer */ + size -= nread; + } + +done: + if(fh > 0) + HDclose(fh); + + return; +} +#endif + diff --git a/tools/src/h5repack/h5repack_filters.c b/tools/src/h5repack/h5repack_filters.c new file mode 100644 index 0000000..e21b829 --- /dev/null +++ b/tools/src/h5repack/h5repack_filters.c @@ -0,0 +1,493 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/* number of members in an array */ +#ifndef NELMTS +# define NELMTS(X) (sizeof(X)/sizeof(X[0])) +#endif + +/* minimum of two values */ +#undef MIN +#define MIN(a,b) (((a)<(b)) ? (a) : (b)) + +/*------------------------------------------------------------------------- + * Function: aux_find_obj + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list + * + *------------------------------------------------------------------------- + */ +static int +aux_find_obj(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ +{ + char *pdest; + int result; + unsigned int i; + + for ( i=0; i<options->op_tbl->nelems; i++) + { + if (HDstrcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj = options->op_tbl->objs[i]; + return (int)i; + } + + pdest = HDstrstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj = options->op_tbl->objs[i]; + return (int)i; + } + }/*i*/ + + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: aux_assign_obj + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the filter information OBJ + * + * Return: 0 not found, 1 found + * + *------------------------------------------------------------------------- + */ +static int +aux_assign_obj(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ +{ + + int idx, i; + pack_info_t tmp; + + init_packobject(&tmp); + + idx = aux_find_obj(name,options,&tmp); + + /* name was on input */ + if (idx>=0) + { + + + /* applying to all objects */ + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + } + else + { + tmp.layout = options->op_tbl->objs[idx].layout; + switch (tmp.layout) + { + case H5D_CHUNKED: + tmp.chunk.rank = options->op_tbl->objs[idx].chunk.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->op_tbl->objs[idx].chunk.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + + } + + /* applying to all objects */ + if (options->all_filter) + { + /* assign the global filter */ + tmp.nfilters=1; + tmp.filter[0]=options->filter_g[0]; + } /* if all */ + else + { + tmp.nfilters=options->op_tbl->objs[idx].nfilters; + for ( i=0; i<tmp.nfilters; i++) + { + tmp.filter[i] = options->op_tbl->objs[idx].filter[i]; + } + } + + + } /* if idx */ + + + /* no input name */ + + else + { + + if (options->all_filter) + { + int k; + + /* assign the global filters */ + tmp.nfilters=options->n_filter_g; + for ( k = 0; k < options->n_filter_g; k++) + tmp.filter[k]=options->filter_g[k]; + } + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + } + } + + *obj = tmp; + return 1; + +} + + +/*------------------------------------------------------------------------- + * Function: apply_filters + * + * Purpose: apply the filters in the object to the property list; + * do extra checking in the case of SZIP; delete all filters in the case + * of H5Z_FILTER_NONE present in the PACK_INFO_T filter array + * + * Return: 0 success, -1 an error occured + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int apply_filters(const char* name, /* object name from traverse list */ + int rank, /* rank of dataset */ + hsize_t *dims, /* dimensions of dataset */ + size_t msize, /* size of type */ + hid_t dcpl_id, /* dataset creation property list */ + pack_opt_t *options, /* repack options */ + int *has_filter) /* (OUT) object NAME has a filter */ + + +{ + int nfilters; /* number of filters in DCPL */ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; + int i; + pack_info_t obj; + + *has_filter = 0; + + if (rank==0) /* scalar dataset, do not apply */ + return 0; + + /*------------------------------------------------------------------------- + * initialize the assigment object + *------------------------------------------------------------------------- + */ + init_packobject(&obj); + + /*------------------------------------------------------------------------- + * find options + *------------------------------------------------------------------------- + */ + if (aux_assign_obj(name,options,&obj)==0) + return 0; + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + /*------------------------------------------------------------------------- + * check if we have filters in the pipeline + * we want to replace them with the input filters + * only remove if we are inserting new ones + *------------------------------------------------------------------------- + */ + if (nfilters && obj.nfilters ) + { + *has_filter = 1; + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } + + /*------------------------------------------------------------------------- + * check if there is an existent chunk + * read it only if there is not a requested layout + *------------------------------------------------------------------------- + */ + if (obj.layout == -1 ) + { + if ((layout = H5Pget_layout(dcpl_id))<0) + return -1; + + if (layout == H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + obj.layout = H5D_CHUNKED; + obj.chunk.rank = rank; + for ( i = 0; i < rank; i++) + obj.chunk.chunk_lengths[i] = chsize[i]; + } + } + + /*------------------------------------------------------------------------- + * the type of filter and additional parameter + * type can be one of the filters + * H5Z_FILTER_NONE 0 , uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression + *------------------------------------------------------------------------- + */ + + if (obj.nfilters) + { + + /*------------------------------------------------------------------------- + * filters require CHUNK layout; if we do not have one define a default + *------------------------------------------------------------------------- + */ + if (obj.layout==-1) + { + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + + obj.chunk.rank = rank; + + /* + * determine the strip mine size. The strip mine is + * a hyperslab whose size is manageable. + */ + + + + sm_nbytes = msize; + for ( i = rank; i > 0; --i) + { + hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; + if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */ + size = 1; + sm_size[i - 1] = MIN(dims[i - 1], size); + sm_nbytes *= sm_size[i - 1]; + HDassert(sm_nbytes > 0); + + } + + for ( i = 0; i < rank; i++) + { + obj.chunk.chunk_lengths[i] = sm_size[i]; + } + + } + + for ( i=0; i<obj.nfilters; i++) + { + switch (obj.filter[i].filtn) + { + + /*------------------------------------------------------------------------- + * H5Z_FILTER_NONE 0 , uncompress if compressed + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_NONE: + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_DEFLATE: + { + unsigned aggression; /* the deflate level */ + + aggression = obj.filter[i].cd_values[0]; + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP 4 , szip compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SZIP: + { + unsigned options_mask; + unsigned pixels_per_block; + + options_mask = obj.filter[i].cd_values[0]; + pixels_per_block = obj.filter[i].cd_values[1]; + + /* set up for szip data */ + if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0) + return -1; + + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SHUFFLE: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_shuffle(dcpl_id)<0) + return -1; + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_FLETCHER32: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_fletcher32(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_NBIT , NBIT compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_NBIT: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_nbit(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET , scale+offset compression + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_SCALEOFFSET: + { + H5Z_SO_scale_type_t scale_type; + int scale_factor; + + scale_type = (H5Z_SO_scale_type_t)obj.filter[i].cd_values[0]; + scale_factor = (int)obj.filter[i].cd_values[1]; + + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0) + return -1; + } + break; + default: + { + if (H5Pset_filter (dcpl_id, obj.filter[i].filtn, H5Z_FLAG_MANDATORY, obj.filter[i].cd_nelmts, obj.filter[i].cd_values)<0) + return -1; + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + } + break; + } /* switch */ + }/*i*/ + + } + /*obj.nfilters*/ + + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + + if (obj.layout>=0) + { + /* a layout was defined */ + if (H5Pset_layout(dcpl_id, obj.layout)<0) + return -1; + + if (H5D_CHUNKED == obj.layout) + { + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + } + else if (H5D_COMPACT == obj.layout) + { + if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0) + return -1; + } + /* remove filters for the H5D_CONTIGUOUS case */ + else if (H5D_CONTIGUOUS == obj.layout) + { + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } + + } + + return 0; +} + diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c new file mode 100644 index 0000000..2d48a04 --- /dev/null +++ b/tools/src/h5repack/h5repack_main.c @@ -0,0 +1,664 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5tools.h" +#include "h5tools_utils.h" +#include "h5repack.h" + +/* Name of tool */ +#define PROGRAMNAME "h5repack" + +static int parse_command_line(int argc, const char **argv, pack_opt_t* options); +static void leave(int ret) H5_ATTR_NORETURN; + + +/* module-scoped variables */ +static int has_i_o = 0; +const char *infile = NULL; +const char *outfile = NULL; + +/* + * Command-line options: The user can specify short or long-named + * parameters. + */ +static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:T:E"; +static struct long_options l_opts[] = { + { "help", no_arg, 'h' }, + { "version", no_arg, 'V' }, + { "verbose", no_arg, 'v' }, + { "filter", require_arg, 'f' }, + { "layout", require_arg, 'l' }, + { "minimum", require_arg, 'm' }, + { "file", require_arg, 'e' }, + { "native", no_arg, 'n' }, + { "latest", no_arg, 'L' }, + { "compact", require_arg, 'c' }, + { "indexed", require_arg, 'd' }, + { "ssize", require_arg, 's' }, + { "ublock", require_arg, 'u' }, + { "block", require_arg, 'b' }, + { "metadata_block_size", require_arg, 'M' }, + { "threshold", require_arg, 't' }, + { "alignment", require_arg, 'a' }, + { "infile", require_arg, 'i' }, /* -i for backward compability */ + { "outfile", require_arg, 'o' }, /* -o for backward compability */ + { "fs_strategy", require_arg, 'S' }, + { "fs_threshold", require_arg, 'T' }, + { "enable-error-stack", no_arg, 'E' }, + { NULL, 0, '\0' } +}; + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: print usage + * + * Return: void + * + *------------------------------------------------------------------------- + */ +static void usage(const char *prog) { + FLUSHSTREAM(rawoutstream); + PRINTSTREAM(rawoutstream, "usage: %s [OPTIONS] file1 file2\n", prog); + PRINTVALSTREAM(rawoutstream, " file1 Input HDF5 File\n"); + PRINTVALSTREAM(rawoutstream, " file2 Output HDF5 File\n"); + PRINTVALSTREAM(rawoutstream, " OPTIONS\n"); + PRINTVALSTREAM(rawoutstream, " -h, --help Print a usage message and exit\n"); + PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n"); + PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); + PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n"); + PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n"); + PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n"); + PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n"); + PRINTVALSTREAM(rawoutstream, " -s S[:F], --ssize=S[:F] Shared object header message minimum size\n"); + PRINTVALSTREAM(rawoutstream, " -m M, --minimum=M Do not apply the filter to datasets smaller than M\n"); + PRINTVALSTREAM(rawoutstream, " -e E, --file=E Name of file E with the -f and -l options\n"); + PRINTVALSTREAM(rawoutstream, " -u U, --ublock=U Name of file U with user block data to be added\n"); + PRINTVALSTREAM(rawoutstream, " -b B, --block=B Size of user block to be added\n"); + PRINTVALSTREAM(rawoutstream, " -M A, --metadata_block_size=A Metadata block size for H5Pset_meta_block_size\n"); + PRINTVALSTREAM(rawoutstream, " -t T, --threshold=T Threshold value for H5Pset_alignment\n"); + PRINTVALSTREAM(rawoutstream, " -a A, --alignment=A Alignment value for H5Pset_alignment\n"); + PRINTVALSTREAM(rawoutstream, " -f FILT, --filter=FILT Filter type\n"); + PRINTVALSTREAM(rawoutstream, " -l LAYT, --layout=LAYT Layout type\n"); + PRINTVALSTREAM(rawoutstream, " -S FS_STRGY, --fs_strategy=FS_STRGY File space management strategy\n"); + PRINTVALSTREAM(rawoutstream, " -T FS_THRD, --fs_threshold=FS_THRD Free-space section threshold\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " M - is an integer greater than 1, size of dataset in bytes (default is 0) \n"); + PRINTVALSTREAM(rawoutstream, " E - is a filename.\n"); + PRINTVALSTREAM(rawoutstream, " S - is an integer\n"); + PRINTVALSTREAM(rawoutstream, " U - is a filename.\n"); + PRINTVALSTREAM(rawoutstream, " T - is an integer\n"); + PRINTVALSTREAM(rawoutstream, " A - is an integer greater than zero\n"); + PRINTVALSTREAM(rawoutstream, " B - is the user block size, any value that is 512 or greater and is\n"); + PRINTVALSTREAM(rawoutstream, " a power of 2 (1024 default)\n"); + PRINTVALSTREAM(rawoutstream, " F - is the shared object header message type, any of <dspace|dtype|fill|\n"); + PRINTVALSTREAM(rawoutstream, " pline|attr>. If F is not specified, S applies to all messages\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n"); + PRINTVALSTREAM(rawoutstream, " occur.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_STRGY is the file space management strategy to use for the output file.\n"); + PRINTVALSTREAM(rawoutstream, " It is a string as listed below:\n"); + PRINTVALSTREAM(rawoutstream, " ALL_PERSIST - Use persistent free-space managers, aggregators and virtual file driver\n"); + PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, " ALL - Use non-persistent free-space managers, aggregators and virtual file driver\n"); + PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, " AGGR_VFD - Use aggregators and virtual file driver for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, " VFD - Use virtual file driver for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_THRD is the free-space section threshold to use for the output file.\n"); + PRINTVALSTREAM(rawoutstream, " It is the minimum size (in bytes) of free-space sections to be tracked\n"); + PRINTVALSTREAM(rawoutstream, " by the the library's free-space managers.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FILT - is a string with the format:\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " <list of objects>:<name of filter>=<filter parameters>\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " <list of objects> is a comma separated list of object names, meaning apply\n"); + PRINTVALSTREAM(rawoutstream, " compression only to those objects. If no names are specified, the filter\n"); + PRINTVALSTREAM(rawoutstream, " is applied to all objects\n"); + PRINTVALSTREAM(rawoutstream, " <name of filter> can be:\n"); + PRINTVALSTREAM(rawoutstream, " GZIP, to apply the HDF5 GZIP filter (GZIP compression)\n"); + PRINTVALSTREAM(rawoutstream, " SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n"); + PRINTVALSTREAM(rawoutstream, " SHUF, to apply the HDF5 shuffle filter\n"); + PRINTVALSTREAM(rawoutstream, " FLET, to apply the HDF5 checksum filter\n"); + PRINTVALSTREAM(rawoutstream, " NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n"); + PRINTVALSTREAM(rawoutstream, " SOFF, to apply the HDF5 Scale/Offset filter\n"); + PRINTVALSTREAM(rawoutstream, " UD, to apply a user defined filter\n"); + PRINTVALSTREAM(rawoutstream, " NONE, to remove all filters\n"); + PRINTVALSTREAM(rawoutstream, " <filter parameters> is optional filter parameter information\n"); + PRINTVALSTREAM(rawoutstream, " GZIP=<deflation level> from 1-9\n"); + PRINTVALSTREAM(rawoutstream, " SZIP=<pixels per block,coding> pixels per block is a even number in\n"); + PRINTVALSTREAM(rawoutstream, " 2-32 and coding method is either EC or NN\n"); + PRINTVALSTREAM(rawoutstream, " SHUF (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, " FLET (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, " NBIT (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, " SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n"); + PRINTVALSTREAM(rawoutstream, " is either IN or DS\n"); + PRINTVALSTREAM(rawoutstream, " UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]>\n"); + PRINTVALSTREAM(rawoutstream, " required values for filter_number,cd_value_count,value_1\n"); + PRINTVALSTREAM(rawoutstream, " optional values for value_2 to value_N\n"); + PRINTVALSTREAM(rawoutstream, " NONE (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " LAYT - is a string with the format:\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " <list of objects>:<layout type>=<layout parameters>\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " <list of objects> is a comma separated list of object names, meaning that\n"); + PRINTVALSTREAM(rawoutstream, " layout information is supplied for those objects. If no names are\n"); + PRINTVALSTREAM(rawoutstream, " specified, the layout type is applied to all objects\n"); + PRINTVALSTREAM(rawoutstream, " <layout type> can be:\n"); + PRINTVALSTREAM(rawoutstream, " CHUNK, to apply chunking layout\n"); + PRINTVALSTREAM(rawoutstream, " COMPA, to apply compact layout\n"); + PRINTVALSTREAM(rawoutstream, " CONTI, to apply contiguous layout\n"); + PRINTVALSTREAM(rawoutstream, " <layout parameters> is optional layout information\n"); + PRINTVALSTREAM(rawoutstream, " CHUNK=DIM[xDIM...xDIM], the chunk size of each dimension\n"); + PRINTVALSTREAM(rawoutstream, " COMPA (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, " CONTI (no parameter)\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Examples of use:\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "1) h5repack -v -f GZIP=1 file1 file2\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " GZIP compression with level 1 to all objects\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "2) h5repack -v -f dset1:SZIP=8,NN file1 file2\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " SZIP compression with 8 pixels per block and NN coding method to object dset1\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "3) h5repack -v -l dset1,dset2:CHUNK=20x10 -f dset3,dset4,dset5:NONE file1 file2\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " Chunked layout, with a layout size of 20x10, to objects dset1 and dset2\n"); + PRINTVALSTREAM(rawoutstream, " and remove filters to objects dset3, dset4, dset5\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "4) h5repack -L -c 10 -s 20:dtype file1 file2 \n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " Using latest file format with maximum compact group size of 10 and\n"); + PRINTVALSTREAM(rawoutstream, " and minimum shared datatype size of 20\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "5) h5repack -f SHUF -f GZIP=1 file1 file2 \n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " Add both filters SHUF and GZIP in this order to all datasets\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "6) h5repack -f UD=307,1,9 file1 file2 \n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " Add bzip2 filter to all datasets\n"); + PRINTVALSTREAM(rawoutstream, "\n"); +} + +/*------------------------------------------------------------------------- + * Function: leave + * + * Purpose: Shutdown MPI & HDF5 and call exit() + * + * Return: Does not return + * + * Programmer: Quincey Koziol + * Saturday, 31. January 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void leave(int ret) { + h5tools_close(); + + HDexit(ret); +} + +/*------------------------------------------------------------------------- + * Function: read_info + * + * Purpose: read comp and chunk options from a file + * + * Return: void, exit on error + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ + +static +int read_info(const char *filename, pack_opt_t *options) { + + char stype[10]; + char comp_info[1024]; + FILE *fp = NULL; + char c; + int i, rc = 1; + int ret_value = EXIT_SUCCESS; + + if ((fp = HDfopen(filename, "r")) == (FILE *) NULL) { + error_msg("cannot open options file %s\n", filename); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } + + /* cycle until end of file reached */ + while (1) { + rc = fscanf(fp, "%s", stype); + if (rc == -1) + break; + + /*------------------------------------------------------------------------- + * filter + *------------------------------------------------------------------------- + */ + if (HDstrcmp(stype,"-f") == 0) { + /* find begining of info */ + i = 0; + c = '0'; + while (c != ' ') { + if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + error_msg("fscanf error\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } /* end if */ + if (HDfeof(fp)) + break; + } + c = '0'; + /* go until end */ + while (c != ' ') { + if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + error_msg("fscanf error\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } /* end if */ + comp_info[i] = c; + i++; + if (HDfeof(fp)) + break; + if (c == 10 /*eol*/) + break; + } + comp_info[i - 1] = '\0'; /*cut the last " */ + + if (h5repack_addfilter(comp_info, options) == -1) { + error_msg("could not add compression option\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } + } + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(stype,"-l") == 0) { + + /* find begining of info */ + i = 0; + c = '0'; + while (c != ' ') { + if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + error_msg("fscanf error\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } /* end if */ + if (HDfeof(fp)) + break; + } + c = '0'; + /* go until end */ + while (c != ' ') { + if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + error_msg("fscanf error\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } /* end if */ + comp_info[i] = c; + i++; + if (HDfeof(fp)) + break; + if (c == 10 /*eol*/) + break; + } + comp_info[i - 1] = '\0'; /*cut the last " */ + + if (h5repack_addlayout(comp_info, options) == -1) { + error_msg("could not add chunck option\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } + } + /*------------------------------------------------------------------------- + * not valid + *------------------------------------------------------------------------- + */ + else { + error_msg("bad file format for %s", filename); + h5tools_setstatus(EXIT_FAILURE); + ret_value = EXIT_FAILURE; + goto done; + } + } + +done: + if (fp) + HDfclose(fp); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: parse_command_line + * + * Purpose: parse command line input + * + *------------------------------------------------------------------------- + */ + +static +int parse_command_line(int argc, const char **argv, pack_opt_t* options) { + + int opt; + int ret_value = 0; + + /* parse command line options */ + while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + switch ((char) opt) { + + /* -i for backward compability */ + case 'i': + infile = opt_arg; + has_i_o = 1; + break; + + /* -o for backward compability */ + case 'o': + outfile = opt_arg; + has_i_o = 1; + break; + + case 'h': + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + ret_value = -1; + goto done; + + case 'V': + print_version(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + ret_value = -1; + goto done; + + case 'v': + options->verbose = 1; + break; + + case 'f': + /* parse the -f filter option */ + if (h5repack_addfilter(opt_arg, options) < 0) { + error_msg("in parsing filter\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + break; + + case 'l': + /* parse the -l layout option */ + if (h5repack_addlayout(opt_arg, options) < 0) { + error_msg("in parsing layout\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + break; + + case 'm': + options->min_comp = HDstrtoull(opt_arg , NULL, 0); + if ((int) options->min_comp <= 0) { + error_msg("invalid minimum compress size <%s>\n", opt_arg); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + break; + + case 'e': + ret_value = read_info(opt_arg, options); + if (ret_value < 0) + goto done; + break; + + case 'n': + options->use_native = 1; + break; + + case 'L': + options->latest = TRUE; + break; + + case 'c': + options->grp_compact = HDatoi( opt_arg ); + if (options->grp_compact > 0) + options->latest = TRUE; /* must use latest format */ + break; + + case 'd': + options->grp_indexed = HDatoi( opt_arg ); + if (options->grp_indexed > 0) + options->latest = TRUE; /* must use latest format */ + break; + + case 's': + { + int idx = 0; + int ssize = 0; + char *msgPtr = HDstrchr( opt_arg, ':'); + options->latest = TRUE; /* must use latest format */ + if (msgPtr == NULL) { + ssize = HDatoi( opt_arg ); + for (idx = 0; idx < 5; idx++) + options->msg_size[idx] = ssize; + } + else { + char msgType[10]; + HDstrcpy(msgType, msgPtr + 1); + msgPtr[0] = '\0'; + ssize = HDatoi( opt_arg ); + if (HDstrncmp(msgType, "dspace",6) == 0) { + options->msg_size[0] = ssize; + } + else if (HDstrncmp(msgType, "dtype", 5) == 0) { + options->msg_size[1] = ssize; + } + else if (HDstrncmp(msgType, "fill", 4) == 0) { + options->msg_size[2] = ssize; + } + else if (HDstrncmp(msgType, "pline", 5) == 0) { + options->msg_size[3] = ssize; + } + else if (HDstrncmp(msgType, "attr", 4) == 0) { + options->msg_size[4] = ssize; + } + } + } + break; + + case 'u': + options->ublock_filename = opt_arg; + break; + + case 'b': + options->ublock_size = (hsize_t) HDatol( opt_arg ); + break; + + case 'M': + options->meta_block_size = (hsize_t) HDatol( opt_arg ); + break; + + case 't': + options->threshold = (hsize_t) HDatol( opt_arg ); + break; + + case 'a': + options->alignment = HDstrtoull(opt_arg , NULL, 0); + if (options->alignment < 1) { + error_msg("invalid alignment size\n", opt_arg); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + break; + + case 'S': + { + char strategy[MAX_NC_NAME]; + + HDstrcpy(strategy, opt_arg); + if (!HDstrcmp(strategy, "ALL_PERSIST")) + options->fs_strategy = H5F_FILE_SPACE_ALL_PERSIST; + else if (!HDstrcmp(strategy, "ALL")) + options->fs_strategy = H5F_FILE_SPACE_ALL; + else if (!HDstrcmp(strategy, "AGGR_VFD")) + options->fs_strategy = H5F_FILE_SPACE_AGGR_VFD; + else if (!HDstrcmp(strategy, "VFD")) + options->fs_strategy = H5F_FILE_SPACE_VFD; + else { + error_msg("invalid file space management strategy\n", opt_arg); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + } + break; + + case 'T': + options->fs_threshold = (hsize_t) HDatol( opt_arg ); + break; + + case 'E': + enable_error_stack = TRUE; + break; + + default: + break; + } /* switch */ + + } /* while */ + + if (has_i_o == 0) { + /* check for file names to be processed */ + if (argc <= opt_ind || argv[opt_ind + 1] == NULL) { + error_msg("missing file names\n"); + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + } + } + +done: + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: h5repack main program + * + * Return: Success: EXIT_SUCCESS(0) + * + * Failure: EXIT_FAILURE(1) + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: May 9, 2003 + * + * Comments: + * + *------------------------------------------------------------------------- + */ +int main(int argc, const char **argv) { + H5E_auto2_t func; + H5E_auto2_t tools_func; + void *edata; + void *tools_edata; + + pack_opt_t options; /*the global options */ + + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + + /* Disable error reporting */ + H5Eget_auto2(H5E_DEFAULT, &func, &edata); + H5Eset_auto2(H5E_DEFAULT, NULL, NULL); + + /* Initialize h5tools lib */ + h5tools_init(); + + /* Disable tools error reporting */ + H5Eget_auto2(H5tools_ERR_STACK_g, &tools_func, &tools_edata); + H5Eset_auto2(H5tools_ERR_STACK_g, NULL, NULL); + + /* update hyperslab buffer size from H5TOOLS_BUFSIZE env if exist */ + if (h5tools_getenv_update_hyperslab_bufsize() < 0) { + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + + /* initialize options */ + h5repack_init(&options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t) 0); + + if (parse_command_line(argc, argv, &options) < 0) + goto done; + + /* get file names if they were not yet got */ + if (has_i_o == 0) { + + if (argv[opt_ind] != NULL && argv[opt_ind + 1] != NULL) { + infile = argv[opt_ind]; + outfile = argv[opt_ind + 1]; + + if ( HDstrcmp( infile, outfile ) == 0) { + error_msg("file names cannot be the same\n"); + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + else { + error_msg("file names missing\n"); + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + + if (enable_error_stack) { + H5Eset_auto2(H5E_DEFAULT, func, edata); + H5Eset_auto2(H5tools_ERR_STACK_g, tools_func, tools_edata); + } + + /* pack it */ + h5tools_setstatus(h5repack(infile, outfile, &options)); + +done: + /* free tables */ + h5repack_end(&options); + + leave(h5tools_getstatus()); +} + diff --git a/tools/src/h5repack/h5repack_opttable.c b/tools/src/h5repack/h5repack_opttable.c new file mode 100644 index 0000000..8c98b76 --- /dev/null +++ b/tools/src/h5repack/h5repack_opttable.c @@ -0,0 +1,367 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/*------------------------------------------------------------------------- + * Function: init_packobject + * + * Purpose: initialize a pack_info_t structure + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +void init_packobject(pack_info_t *obj) { + int j, k; + + HDstrcpy(obj->path, "\0"); + for (j = 0; j < H5_REPACK_MAX_NFILTERS; j++) { + obj->filter[j].filtn = -1; + for (k = 0; k < CD_VALUES; k++) + obj->filter[j].cd_values[k] = 0; + } + obj->chunk.rank = -1; + obj->refobj_id = -1; + obj->layout = H5D_LAYOUT_ERROR; + obj->nfilters = 0; +} + +/*------------------------------------------------------------------------- + * Function: aux_tblinsert_filter + * + * Purpose: auxiliary function, inserts the filter in object OBJS[ I ] + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +static void aux_tblinsert_filter(pack_opttbl_t *table, unsigned int I, + filter_info_t filt) { + if (table->objs[I].nfilters < H5_REPACK_MAX_NFILTERS) { + table->objs[I].filter[table->objs[I].nfilters++] = filt; + } + else { + error_msg( + "cannot insert the filter in this object.\ + Maximum capacity exceeded\n"); + } +} + +/*------------------------------------------------------------------------- + * Function: aux_tblinsert_layout + * + * Purpose: auxiliary function, inserts the layout in object OBJS[ I ] + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +static void aux_tblinsert_layout(pack_opttbl_t *table, unsigned int I, + pack_info_t *pack) { + int k; + + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED == pack->layout) { + /* -2 means the NONE option, remove chunking + and set the layout to contiguous */ + if (pack->chunk.rank == -2) { + table->objs[I].layout = H5D_CONTIGUOUS; + table->objs[I].chunk.rank = -2; + } + /* otherwise set the chunking type */ + else { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = + pack->chunk.chunk_lengths[k]; + } + } +} + +/*------------------------------------------------------------------------- + * Function: aux_inctable + * + * Purpose: auxiliary function, increases the size of the collection by N_OBJS + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +static int +aux_inctable(pack_opttbl_t *table, unsigned n_objs) +{ + unsigned u; + + table->size += n_objs; + table->objs = (pack_info_t*) HDrealloc(table->objs, table->size * sizeof(pack_info_t)); + if (table->objs == NULL) { + error_msg("not enough memory for options table\n"); + return -1; + } + + for (u = table->nelems; u < table->size; u++) + init_packobject(&table->objs[u]); + + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: options_table_init + * + * Purpose: init options table + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int options_table_init(pack_opttbl_t **tbl) { + unsigned int i; + pack_opttbl_t *table; + + if (NULL == (table = (pack_opttbl_t *) HDmalloc(sizeof(pack_opttbl_t)))) { + error_msg("not enough memory for options table\n"); + return -1; + } + + table->size = 30; + table->nelems = 0; + if (NULL == (table->objs = + (pack_info_t*) HDmalloc(table->size * sizeof(pack_info_t)))) { + error_msg("not enough memory for options table\n"); + HDfree(table); + return -1; + } + + for (i = 0; i < table->size; i++) + init_packobject(&table->objs[i]); + + *tbl = table; + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: options_table_free + * + * Purpose: free table memory + * + * Return: 0 + * + *------------------------------------------------------------------------- + */ + +int options_table_free(pack_opttbl_t *table) { + HDfree(table->objs); + HDfree(table); + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_add_layout + * + * Purpose: add a layout option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int +options_add_layout(obj_list_t *obj_list, unsigned n_objs, pack_info_t *pack, + pack_opttbl_t *table) +{ + unsigned i, j, I; + unsigned added = 0; + hbool_t found = FALSE; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems + n_objs >= table->size) + if (aux_inctable(table, n_objs) < 0) + return -1; + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems > 0) { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) { + /* linear table search */ + for (i = 0; i < table->nelems; i++) { + /*already on the table */ + if (HDstrcmp(obj_list[j].obj,table->objs[i].path) == 0) { + /* already chunk info inserted for this one; exit */ + if (table->objs[i].chunk.rank > 0) { + error_msg("chunk information already inserted for <%s>\n", obj_list[j].obj); + HDexit(EXIT_FAILURE); + } + /* insert the layout info */ + else { + aux_tblinsert_layout(table, i, pack); + found = TRUE; + break; + } + } /* if */ + } /* i */ + + if (!found) { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_layout(table, I, pack); + } + /* cases where we have an already inserted name but there is a new name also + example: + -f dset1:GZIP=1 -l dset1,dset2:CHUNK=20x20 + dset1 is already inserted, but dset2 must also be + */ + else + if(found && HDstrcmp(obj_list[j].obj,table->objs[i].path) != 0) { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_layout(table, I, pack); + } + } /* j */ + } + /* first time insertion */ + else { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) { + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_layout(table, I, pack); + } + } + + table->nelems += added; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_add_filter + * + * Purpose: add a compression -f option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ +int +options_add_filter(obj_list_t *obj_list, unsigned n_objs, filter_info_t filt, + pack_opttbl_t *table) +{ + unsigned int i, j, I; + unsigned added = 0; + hbool_t found = FALSE; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems + n_objs >= table->size) + if (aux_inctable(table, n_objs) < 0) + return -1; + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems > 0) { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) { + /* linear table search */ + for (i = 0; i < table->nelems; i++) { + /*already on the table */ + if (HDstrcmp(obj_list[j].obj, table->objs[i].path) == 0) { + /* insert */ + aux_tblinsert_filter(table, i, filt); + found = TRUE; + break; + } /* if */ + } /* i */ + + if (!found) { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_filter(table, I, filt); + } + /* cases where we have an already inserted name but there is a new name also + example: + -l dset1:CHUNK=20x20 -f dset1,dset2:GZIP=1 + dset1 is already inserted, but dset2 must also be + */ + else + if(found && HDstrcmp(obj_list[j].obj,table->objs[i].path) != 0) { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_filter(table, I, filt); + } + } /* j */ + } + + /* first time insertion */ + else { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) { + I = table->nelems + added; + added++; + HDstrcpy(table->objs[I].path, obj_list[j].obj); + aux_tblinsert_filter(table, I, filt); + } + } + + table->nelems += added; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_get_object + * + * Purpose: get object from table; "path" is the key + * + * Return: pack_info_t* OBJECT or NULL if not found; PATH is the key + * + *------------------------------------------------------------------------- + */ + +pack_info_t* options_get_object(const char *path, pack_opttbl_t *table) { + unsigned int i; + char tbl_path[MAX_NC_NAME + 1]; /* +1 for start with "/" case */ + + for (i = 0; i < table->nelems; i++) { + /* make full path (start with "/") to compare correctly */ + if (HDstrncmp(table->objs[i].path, "/", 1)) { + HDstrcpy(tbl_path, "/"); + HDstrcat(tbl_path, table->objs[i].path); + } + else + HDstrcpy(tbl_path, table->objs[i].path); + + /* found it */ + if (HDstrcmp(tbl_path, path) == 0) { + return (&table->objs[i]); + } + } + + return NULL; +} + diff --git a/tools/src/h5repack/h5repack_parse.c b/tools/src/h5repack/h5repack_parse.c new file mode 100644 index 0000000..004b9e4 --- /dev/null +++ b/tools/src/h5repack/h5repack_parse.c @@ -0,0 +1,616 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/*------------------------------------------------------------------------- + * Function: parse_filter + * + * Purpose: read filter information + * + * Return: a list of names, the number of names and its compression type + * + * <name of filter> can be: + * GZIP, to apply the HDF5 GZIP filter (GZIP compression) + * SZIP, to apply the HDF5 SZIP filter (SZIP compression) + * SHUF, to apply the HDF5 shuffle filter + * FLET, to apply the HDF5 checksum filter + * NBIT, to apply the HDF5 NBIT filter (NBIT compression) + * SOFF, to apply the HDF5 scale+offset filter (compression) + * UD, to apply a User Defined filter k,m,n1[,…,nm] + * NONE, to remove the filter + * + * Examples: + * "GZIP=6" + * "A,B:NONE" + *------------------------------------------------------------------------- + */ +obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt, + pack_opt_t *options, int *is_glb) { + size_t i, m, u; + char c; + size_t len = HDstrlen(str); + int k, l, p, q, end_obj = -1, no_param = 0; + unsigned j, n; + char sobj[MAX_NC_NAME]; + char scomp[10]; + char stype[6]; + char smask[3]; + obj_list_t* obj_list = NULL; + unsigned pixels_per_block; + + /* initialize compression info */ + HDmemset(filt, 0, sizeof(filter_info_t)); + *is_glb = 0; + + /* check for the end of object list and number of objects */ + for (i = 0, n = 0; i < len; i++) { + c = str[i]; + if (c == ':') + end_obj = (int) i; + if (c == ',') + n++; + } + + /* Check for missing : */ + if (end_obj == -1) { + /* apply to all objects */ + options->all_filter = 1; + *is_glb = 1; + } + + n++; + obj_list = (obj_list_t *) HDmalloc(n * sizeof(obj_list_t)); + if (obj_list == NULL) { + error_msg("could not allocate object list\n"); + return NULL; + } + *n_objs = n; + + /* get object list */ + if (end_obj > 0) + for (j = 0, k = 0, n = 0; j < (unsigned) end_obj; j++, k++) { + c = str[j]; + sobj[k] = c; + if (c == ',' || j == (unsigned) (end_obj - 1)) { + if (c == ',') + sobj[k] = '\0'; + else + sobj[k + 1] = '\0'; + HDstrcpy(obj_list[n].obj, sobj); + HDmemset(sobj, 0, sizeof(sobj)); + n++; + k = -1; + } + } + /* nothing after : */ + if (end_obj + 1 == (int) len) { + if (obj_list) + HDfree(obj_list); + error_msg("input Error: Invalid compression type in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + + /* get filter additional parameters */ + m = 0; + for (i = (size_t)(end_obj + 1), k = 0, j = 0; i < len; i++, k++) { + c = str[i]; + scomp[k] = c; + if (c == '=' || i == len - 1) { + if (c == '=') { /*one more parameter */ + scomp[k] = '\0'; /*cut space */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + * szip has the format SZIP=<pixels per block,coding> + * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN' + * example SZIP=8,NN + *------------------------------------------------------------------------- + */ + if (HDstrcmp(scomp, "SZIP") == 0) { + l = -1; /* mask index check */ + for (m = 0, u = i + 1; u < len; u++, m++) { + if (str[u] == ',') { + stype[m] = '\0'; /* end digit of szip */ + l = 0; /* start EC or NN search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!HDisdigit(c) && l == -1) { + if (obj_list) + HDfree(obj_list); + error_msg("compression parameter not digit in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + if (l == -1) + stype[m] = c; + else { + smask[l] = c; + l++; + if (l == 2) { + smask[l] = '\0'; + i = len - 1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (HDstrcmp(smask,"NN") == 0) + filt->cd_values[j++] = H5_SZIP_NN_OPTION_MASK; + else if (HDstrcmp(smask,"EC") == 0) + filt->cd_values[j++] = H5_SZIP_EC_OPTION_MASK; + else { + error_msg("szip mask must be 'NN' or 'EC' \n"); + HDexit(EXIT_FAILURE); + } + } + } + } /* u */ + } /*if */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + * scaleoffset has the format SOFF=<scale_factor,scale_type> + * scale_type can be + * integer datatype, H5Z_SO_INT (IN) + * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) + * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented + * for integer datatypes, scale_factor denotes Minimum Bits + * for float datatypes, scale_factor denotes decimal scale factor + * examples + * SOFF=31,IN + * SOFF=3,DF + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "SOFF") == 0) { + l = -1; /* mask index check */ + for (m = 0, u = i + 1; u < len; u++, m++) { + if (str[u] == ',') { + stype[m] = '\0'; /* end digit */ + l = 0; /* start 'IN' , 'DS', or 'ES' search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!HDisdigit(c) && l == -1) { + if (obj_list) + HDfree(obj_list); + error_msg("compression parameter is not a digit in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + if (l == -1) + stype[m] = c; + else { + smask[l] = c; + l++; + if (l == 2) { + smask[l] = '\0'; + i = len - 1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (HDstrcmp(smask,"IN") == 0) + filt->cd_values[j++] = H5Z_SO_INT; + else if (HDstrcmp(smask, "DS") == H5Z_SO_FLOAT_DSCALE) + filt->cd_values[j++] = H5Z_SO_FLOAT_DSCALE; + else { + error_msg("scale type must be 'IN' or 'DS' \n"); + HDexit(EXIT_FAILURE); + } + } + } + } /* u */ + } /*if */ + + /*------------------------------------------------------------------------- + * User Defined + * has the format UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]> + * BZIP2 example + * UD=307,1,9 + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "UD") == 0) { + l = -1; /* filter number index check */ + p = -1; /* CD_VAL count check */ + for (m = 0, q = 0, u = i + 1; u < len; u++, m++, q++) { + if (str[u] == ',') { + stype[q] = '\0'; /* end digit */ + if (l == -1) { + filt->filtn = HDatoi(stype); + l = 0; + } + else if (p == -1) { + filt->cd_nelmts = HDstrtoull(stype, NULL, 0); + p = 0; + } + else + filt->cd_values[j++] = (unsigned)HDstrtoul(stype, NULL, 0); + q = 0; + u++; /* skip ',' */ + } + c = str[u]; + if (!HDisdigit(c) && l == -1) { + if (obj_list) + HDfree(obj_list); + error_msg("filter number parameter is not a digit in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + stype[q] = c; + } /* for u */ + stype[q] = '\0'; + } /*if */ + + /*------------------------------------------------------------------------- + * all other filters + *------------------------------------------------------------------------- + */ + else { + /* here we could have 1 or 2 digits */ + for (m = 0, u = i + 1; u < len; u++, m++) { + c = str[u]; + if (!HDisdigit(c)) { + if (obj_list) + HDfree(obj_list); + error_msg("compression parameter is not a digit in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + stype[m] = c; + } /* u */ + + stype[m] = '\0'; + } /*if */ + + filt->cd_values[j++] = (unsigned) HDstrtoul(stype, NULL, 0); + if(filt->cd_nelmts == 0) + j = 0; + i += m; /* jump */ + } + else if (i == len - 1) { /*no more parameters */ + scomp[k + 1] = '\0'; + no_param = 1; + } + + /*------------------------------------------------------------------------- + * translate from string to filter symbol + *------------------------------------------------------------------------- + */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_NONE + *------------------------------------------------------------------------- + */ + if (HDstrcmp(scomp, "NONE") == 0) { + filt->filtn = H5Z_FILTER_NONE; + filt->cd_nelmts = 0; + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "GZIP") == 0) { + filt->filtn = H5Z_FILTER_DEFLATE; + filt->cd_nelmts = 1; + if (no_param) { /*no more parameters, GZIP must have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("missing compression parameter in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "SZIP") == 0) { + filt->filtn = H5Z_FILTER_SZIP; + filt->cd_nelmts = 2; + if (no_param) { /*no more parameters, SZIP must have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("missing compression parameter in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "SHUF") == 0) { + filt->filtn = H5Z_FILTER_SHUFFLE; + filt->cd_nelmts = 0; + if (m > 0) { /*shuffle does not have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("extra parameter in SHUF <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "FLET") == 0) { + filt->filtn = H5Z_FILTER_FLETCHER32; + filt->cd_nelmts = 0; + if (m > 0) { /*shuffle does not have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("extra parameter in FLET <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + /*------------------------------------------------------------------------- + * H5Z_FILTER_NBIT + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "NBIT") == 0) { + filt->filtn = H5Z_FILTER_NBIT; + filt->cd_nelmts = 0; + if (m > 0) { /*nbit does not have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("extra parameter in NBIT <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "SOFF") == 0) { + filt->filtn = H5Z_FILTER_SCALEOFFSET; + filt->cd_nelmts = 2; + if (no_param) { /*no more parameters, SOFF must have parameter */ + if (obj_list) + HDfree(obj_list); + error_msg("missing compression parameter in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + /*------------------------------------------------------------------------- + * User Defined Filter + *------------------------------------------------------------------------- + */ + else if (HDstrcmp(scomp, "UD") == 0) { + /* parameters does not match count */ + if (filt->cd_nelmts != j) { + if (obj_list) + HDfree(obj_list); + error_msg("incorrect number of compression parameters in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + else { + if (obj_list) + HDfree(obj_list); + error_msg("invalid filter type in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + } /*i*/ + + /*------------------------------------------------------------------------- + * check valid parameters + *------------------------------------------------------------------------- + */ + + switch (filt->filtn) { + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_DEFLATE: + if (filt->cd_values[0] > 9) { + if (obj_list) + HDfree(obj_list); + error_msg("invalid compression parameter in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + break; + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SZIP: + pixels_per_block = filt->cd_values[0]; + if ((pixels_per_block % 2) == 1) { + if (obj_list) + HDfree(obj_list); + error_msg("pixels_per_block is not even in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + if (pixels_per_block > H5_SZIP_MAX_PIXELS_PER_BLOCK) { + if (obj_list) + HDfree(obj_list); + error_msg("pixels_per_block is too large in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + if ((HDstrcmp(smask,"NN") != 0) && (HDstrcmp(smask,"EC") != 0)) { + if (obj_list) + HDfree(obj_list); + error_msg("szip mask must be 'NN' or 'EC' \n"); + HDexit(EXIT_FAILURE); + } + break; + default: + break; + }; + + return obj_list; +} + + +/*------------------------------------------------------------------------- + * Function: parse_layout + * + * Purpose: read layout info + * + * Return: a list of names, the number of names and its chunking info for + * chunked. NULL, on error + * the layout type can be: + * CHUNK, to apply chunking layout + * CONTI, to apply contiguous layout + * COMPA, to apply compact layout + * + * Example: + * "AA,B,CDE:CHUNK=10X10" + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ +obj_list_t* parse_layout(const char *str, unsigned *n_objs, pack_info_t *pack, /* info about layout needed */ +pack_opt_t *options) { + obj_list_t* obj_list = NULL; + unsigned i, j, n; + char c; + size_t len = HDstrlen(str); + int k, end_obj = -1, c_index; + char sobj[MAX_NC_NAME]; + char sdim[10]; + char slayout[10]; + + HDmemset(sdim, '\0', sizeof(sdim)); + HDmemset(sobj, '\0', sizeof(sobj)); + HDmemset(slayout, '\0', sizeof(slayout)); + + /* check for the end of object list and number of objects */ + for (i = 0, n = 0; i < len; i++) { + c = str[i]; + if (c == ':') + end_obj = (int) i; + if (c == ',') + n++; + } + + if (end_obj == -1) { /* missing : chunk all */ + options->all_layout = 1; + } + + n++; + obj_list = (obj_list_t*) HDmalloc(n * sizeof(obj_list_t)); + if (obj_list == NULL) { + error_msg("could not allocate object list\n"); + return NULL; + } + *n_objs = n; + + /* get object list */ + if (end_obj > 0) + for (j = 0, k = 0, n = 0; j < (unsigned) end_obj; j++, k++) { + c = str[j]; + sobj[k] = c; + if (c == ',' || j == (unsigned) (end_obj - 1)) { + if (c == ',') + sobj[k] = '\0'; + else + sobj[k + 1] = '\0'; + HDstrcpy(obj_list[n].obj, sobj); + HDmemset(sobj, 0, sizeof(sobj)); + n++; + k = -1; + } + } + + /* nothing after : */ + if (end_obj + 1 == (int) len) { + if (obj_list) + HDfree(obj_list); + error_msg("in parse layout, no characters after : in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + + /* get layout info */ + for (j = (unsigned) (end_obj + 1), n = 0; n <= 5; j++, n++) { + if (n == 5) { + slayout[n] = '\0'; /*cut string */ + if (HDstrcmp(slayout, "COMPA") == 0) + pack->layout = H5D_COMPACT; + else if (HDstrcmp(slayout, "CONTI") == 0) + pack->layout = H5D_CONTIGUOUS; + else if (HDstrcmp(slayout, "CHUNK") == 0) + pack->layout = H5D_CHUNKED; + else { + error_msg("in parse layout, not a valid layout in <%s>\n", str); + HDexit(EXIT_FAILURE); + } + } + else { + c = str[j]; + slayout[n] = c; + } + } /* j */ + + if (pack->layout == H5D_CHUNKED) { + /*------------------------------------------------------------------------- + * get chunk info + *------------------------------------------------------------------------- + */ + k = 0; + if (j > len) { + if (obj_list) + HDfree(obj_list); + error_msg("in parse layout, <%s> Chunk dimensions missing\n", str); + HDexit(EXIT_FAILURE); + } + + for (i = j, c_index = 0; i < len; i++) { + c = str[i]; + sdim[k] = c; + k++; /*increment sdim index */ + + if (!HDisdigit(c) && c != 'x' && c != 'N' && c != 'O' && c != 'N' && c != 'E') { + if (obj_list) + HDfree(obj_list); + error_msg("in parse layout, <%s> Not a valid character in <%s>\n", sdim, str); + HDexit(EXIT_FAILURE); + } + + if (c == 'x' || i == len - 1) { + if (c == 'x') { + sdim[k - 1] = '\0'; + k = 0; + pack->chunk.chunk_lengths[c_index] = HDstrtoull(sdim, NULL, 0); + if (pack->chunk.chunk_lengths[c_index] == 0) { + if (obj_list) + HDfree(obj_list); + error_msg("in parse layout, <%s> conversion to number in <%s>\n", sdim, str); + HDexit(EXIT_FAILURE); + } + c_index++; + } + else if (i == len - 1) { /*no more parameters */ + sdim[k] = '\0'; + k = 0; + if (HDstrcmp(sdim,"NONE") == 0) { + pack->chunk.rank = -2; + } + else { + pack->chunk.chunk_lengths[c_index] = HDstrtoull(sdim, NULL, 0); + if (pack->chunk.chunk_lengths[c_index] == 0) { + if (obj_list) + HDfree(obj_list); + error_msg("in parse layout, <%s> conversion to number in <%s>\n", sdim, str); + HDexit(EXIT_FAILURE); + } + pack->chunk.rank = c_index + 1; + } + } /*if */ + } /*if c=='x' || i==len-1 */ + } /*i*/ + } /*H5D_CHUNKED*/ + + return obj_list; +} diff --git a/tools/src/h5repack/h5repack_refs.c b/tools/src/h5repack/h5repack_refs.c new file mode 100644 index 0000000..f0f32c3 --- /dev/null +++ b/tools/src/h5repack/h5repack_refs.c @@ -0,0 +1,877 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5diff.h" +#include "h5tools.h" + + +/*------------------------------------------------------------------------- + * local functions + *------------------------------------------------------------------------- + */ + +static const char* MapIdToName(hid_t refobj_id,trav_table_t *travt); +static int copy_refs_attr(hid_t loc_in, hid_t loc_out, pack_opt_t *options, + trav_table_t *travt, hid_t fidout); +static herr_t update_ref_value(hid_t obj_id, H5R_type_t ref_type, void *ref_in, + hid_t fid_out, void *ref_out, trav_table_t *travt); + +/*------------------------------------------------------------------------- + * Function: do_copy_refobjs + * + * Purpose: duplicate all referenced HDF5 objects in the file + * and create hard links + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December, 10, 2003 + * + *------------------------------------------------------------------------- + */ + +int do_copy_refobjs(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options) /* repack options */ +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t grp_in = (-1); /* read group ID */ + hid_t grp_out = (-1); /* write group ID */ + hid_t dset_in = (-1); /* read dataset ID */ + hid_t dset_out = (-1); /* write dataset ID */ + hid_t type_in = (-1); /* named type ID */ + hid_t dcpl_id = (-1); /* dataset creation property list ID */ + hid_t space_id = (-1); /* space ID */ + hid_t ftype_id = (-1); /* file data type ID */ + hid_t mtype_id = (-1); /* memory data type ID */ + size_t msize; /* memory size of memory type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ + unsigned int i, j; + int k; + named_dt_t *named_dt_head=NULL; /* Pointer to the stack of named datatypes + copied */ + + /*------------------------------------------------------------------------- + * browse + *------------------------------------------------------------------------- + */ + for(i = 0; i < travt->nobjs; i++) { + switch(travt->objs[i].type) + { + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_GROUP + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_GROUP: + /*------------------------------------------------------------------------- + * copy referenced objects in attributes + *------------------------------------------------------------------------- + */ + if((grp_out = H5Gopen2(fidout, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + + if((grp_in = H5Gopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + + if(copy_refs_attr(grp_in, grp_out, options, travt, fidout) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_refs_attr failed"); + + if(H5Gclose(grp_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + if(H5Gclose(grp_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + + /*------------------------------------------------------------------------- + * check for hard links + *------------------------------------------------------------------------- + */ + if(travt->objs[i].nlinks) + for(j = 0; j < travt->objs[i].nlinks; j++) + H5Lcreate_hard(fidout, travt->objs[i].name, H5L_SAME_LOC, travt->objs[i].links[j].new_name, H5P_DEFAULT, H5P_DEFAULT); + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_DATASET + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_DATASET: + if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if((space_id = H5Dget_space(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); + if((ftype_id = H5Dget_type(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); + if((dcpl_id = H5Dget_create_plist(dset_in)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + if((rank = H5Sget_simple_extent_ndims(space_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed"); + if(H5Sget_simple_extent_dims(space_id, dims, NULL) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed"); + nelmts = 1; + for(k = 0; k < rank; k++) + nelmts *= dims[k]; + + if((mtype_id = h5tools_get_native_type(ftype_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5tools_get_native_type failed"); + + if((msize = H5Tget_size(mtype_id)) == 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); + + /*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- + */ + if(h5tools_canreadf(NULL, dcpl_id) == 1) { + /*------------------------------------------------------------------------- + * test for a valid output dataset + *------------------------------------------------------------------------- + */ + dset_out = FAIL; + + /*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + *------------------------------------------------------------------------- + */ + if(H5Tequal(mtype_id, H5T_STD_REF_OBJ)) { + hid_t refobj_id; + hobj_ref_t *refbuf = NULL; /* buffer for object references */ + hobj_ref_t *buf = NULL; + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + if(nelmts) { + buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize)); + if(buf==NULL) { + printf("cannot read into memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dread failed"); + + refbuf = (hobj_ref_t*) HDcalloc((unsigned)nelmts, msize); + if(refbuf == NULL){ + printf("cannot allocate memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); + } /* end if */ + for(u = 0; u < nelmts; u++) { + H5E_BEGIN_TRY { + if((refobj_id = H5Rdereference2(dset_in, H5P_DEFAULT, H5R_OBJECT, &buf[u])) < 0) + continue; + } H5E_END_TRY; + + /* get the name. a valid name could only occur + * in the second traversal of the file + */ + if((refname = MapIdToName(refobj_id, travt)) != NULL) { + /* create the reference, -1 parameter for objects */ + if(H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, (hid_t)-1) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rcreate failed"); + if(options->verbose) + { + printf(FORMAT_OBJ,"dset",travt->objs[i].name ); + printf("object <%s> object reference created to <%s>\n", + travt->objs[i].name, + refname); + } + } /*refname*/ + H5Oclose(refobj_id); + } /* u */ + } /*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, mtype_id, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed"); + if(nelmts) + if(H5Dwrite(dset_out, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, refbuf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dwrite failed"); + + if(buf) + HDfree(buf); + if(refbuf) + HDfree(refbuf); + + /*------------------------------------------------------ + * copy attrs + *----------------------------------------------------*/ + if(copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + } /*H5T_STD_REF_OBJ*/ + + /*------------------------------------------------------------------------- + * dataset region references + *------------------------------------------------------------------------- + */ + else if(H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + { + hid_t refobj_id; + hdset_reg_ref_t *refbuf = NULL; /* input buffer for region references */ + hdset_reg_ref_t *buf = NULL; /* output buffer */ + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + if(nelmts) { + buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize)); + if(buf == NULL) { + printf("cannot read into memory\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dread failed"); + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */ + if(refbuf == NULL) { + printf("cannot allocate memory\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); + } /* end if */ + + for(u = 0; u < nelmts; u++) { + H5E_BEGIN_TRY { + if((refobj_id = H5Rdereference2(dset_in, H5P_DEFAULT, H5R_DATASET_REGION, &buf[u])) < 0) + continue; + } H5E_END_TRY; + + /* get the name. a valid name could only occur + * in the second traversal of the file + */ + if((refname = MapIdToName(refobj_id, travt)) != NULL) { + hid_t region_id; /* region id of the referenced dataset */ + + if((region_id = H5Rget_region(dset_in, H5R_DATASET_REGION, &buf[u])) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rget_region failed"); + + /* create the reference, we need the space_id */ + if(H5Rcreate(&refbuf[u], fidout, refname, H5R_DATASET_REGION, region_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rcreate failed"); + if(H5Sclose(region_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if(options->verbose) + { + printf(FORMAT_OBJ,"dset",travt->objs[i].name ); + printf("object <%s> region reference created to <%s>\n", + travt->objs[i].name, + refname); + } + } /*refname*/ + H5Oclose(refobj_id); + } /* u */ + } /*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, mtype_id, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed"); + if(nelmts) + if(H5Dwrite(dset_out, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, refbuf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dwrite failed"); + + if(buf) + HDfree(buf); + if(refbuf) + HDfree(refbuf); + + /*----------------------------------------------------- + * copy attrs + *----------------------------------------------------*/ + if(copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed"); + } /* H5T_STD_REF_DSETREG */ + /*------------------------------------------------------------------------- + * not references, open previously created object in 1st traversal + *------------------------------------------------------------------------- + */ + else { + if((dset_out = H5Dopen2(fidout, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + } /* end else */ + + HDassert(dset_out != FAIL); + + /*------------------------------------------------------------------------- + * copy referenced objects in attributes + *------------------------------------------------------------------------- + */ + if(copy_refs_attr(dset_in, dset_out, options, travt, fidout) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_refs_attr failed"); + + /*------------------------------------------------------------------------- + * check for hard links + *------------------------------------------------------------------------- + */ + if(travt->objs[i].nlinks) + for(j = 0; j < travt->objs[i].nlinks; j++) + H5Lcreate_hard(fidout, travt->objs[i].name, H5L_SAME_LOC, travt->objs[i].links[j].new_name, H5P_DEFAULT, H5P_DEFAULT); + + if(H5Dclose(dset_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + } /*can_read*/ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Tclose(ftype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if(H5Tclose(mtype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if(H5Pclose(dcpl_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if(H5Sclose(space_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if(H5Dclose(dset_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_NAMED_DATATYPE + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_NAMED_DATATYPE: + if((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Topen2 failed"); + if(H5Tclose(type_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + break; + + /*------------------------------------------------------------------------- + * H5TRAV_TYPE_LINK + *------------------------------------------------------------------------- + */ + case H5TRAV_TYPE_LINK: + /*nothing to do */ + break; + + case H5TRAV_TYPE_UNKNOWN: + case H5TRAV_TYPE_UDLINK: + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5TRAV invalid type"); + + default: + break; + } /* end switch */ + } /* end for */ + + /* Finalize (link) the stack of named datatypes (if any) + * This function is paired with copy_named_datatype() which is called + * in copy_attr(), so need to free. + */ + named_datatype_free(&named_dt_head, 0); + + return ret_value; + +done: + H5E_BEGIN_TRY { + H5Gclose(grp_in); + H5Gclose(grp_out); + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_in); + H5Dclose(dset_out); + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Tclose(type_in); + named_datatype_free(&named_dt_head, 0); + } H5E_END_TRY; + + return ret_value; +} + + +/*------------------------------------------------------------------------- + * Function: copy_refs_attr + * + * Purpose: duplicate all referenced HDF5 located in attributes + * relative to LOC_IN, which is obtained either from + * loc_id = H5Gopen2(fid, name, H5P_DEFAULT); + * loc_id = H5Dopen2(fid, name, H5P_DEFAULT); + * loc_id = H5Topen2(fid, name, H5P_DEFAULT); + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Modifier: xcao@hdfgroup.org, 9/12/2011 + * Update values of references(object and region) for the following types: + * 1) References, + * 2) ARRAY of reference, + * 3) VLEN of references. + * 4) COMPOUND of references. + * This function does not handle references in other complicated structures, + * such as references in nested compound datatypes. + * + * Date: October, 28, 2003 + * + *------------------------------------------------------------------------- + */ + +static int copy_refs_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options, + trav_table_t *travt, + hid_t fidout /* for saving references */ + ) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t attr_id = -1; /* attr ID */ + hid_t attr_out = -1; /* attr ID */ + hid_t space_id = -1; /* space ID */ + hid_t ftype_id = -1; /* file data type ID */ + hid_t mtype_id = -1; /* memory data type ID */ + size_t msize; /* memory size of type */ + hsize_t nelmts; /* number of elements in dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + H5O_info_t oinfo; /* Object info */ + unsigned u, i, j; + int rank; + H5T_class_t type_class = -1; + hbool_t is_ref=0, is_ref_vlen=0, is_ref_array=0, is_ref_comp=0; + void *refbuf = NULL; + void *buf = NULL; + const char* refname = NULL; + unsigned *ref_comp_index = NULL; + size_t *ref_comp_size = NULL; + int ref_comp_field_n = 0; + + + if(H5Oget_info(loc_in, &oinfo) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Oget_info failed"); + + for(u = 0; u < (unsigned)oinfo.num_attrs; u++) { + is_ref = is_ref_vlen = is_ref_array = is_ref_comp = 0; + + /* open attribute */ + if((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)u, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aopen_by_idx failed"); + + /* get the file datatype */ + if((ftype_id = H5Aget_type(attr_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_type failed"); + + type_class = H5Tget_class(ftype_id); + + if((mtype_id = h5tools_get_native_type(ftype_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5tools_get_native_type failed"); + + if((msize = H5Tget_size(mtype_id)) == 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); + + is_ref = (type_class == H5T_REFERENCE); + + if(type_class == H5T_VLEN ) { + hid_t base_type; + + base_type = H5Tget_super(ftype_id); + is_ref_vlen = (H5Tget_class(base_type)==H5T_REFERENCE); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + } + else if(type_class == H5T_ARRAY ) { + hid_t base_type; + + base_type = H5Tget_super(ftype_id); + is_ref_array = (H5Tget_class(base_type)==H5T_REFERENCE); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + } + else if(type_class == H5T_COMPOUND) { + int nmembers = H5Tget_nmembers(ftype_id) ; + + if (nmembers < 1) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_nmembers failed"); + + ref_comp_index = (unsigned *)HDmalloc((size_t)nmembers*sizeof(unsigned)); + ref_comp_size = (size_t *)HDmalloc((size_t)nmembers*sizeof(ref_comp_size)); + ref_comp_field_n = 0; + + for (i=0; i<(unsigned)nmembers; i++) { + hid_t mtid = H5Tget_member_type( ftype_id, i ); + + if ((H5Tget_class(mtid)==H5T_REFERENCE)) { + ref_comp_index[ref_comp_field_n] = i; + ref_comp_size[ref_comp_field_n] = H5Tget_size(mtid); + ref_comp_field_n++; + } + H5Tclose(mtid); + } + + /* if compound don't contain reference type member, free the above + * mallocs. Otherwise there can be memory leaks by the 'continue' + * statement below. */ + if (!ref_comp_field_n) { + if (ref_comp_index) { + HDfree(ref_comp_index); + ref_comp_index = NULL; + } + + if (ref_comp_size) { + HDfree(ref_comp_size); + ref_comp_size = NULL; + } + } + } + + is_ref_comp = (ref_comp_field_n > 0); + + if (!(is_ref || is_ref_vlen || is_ref_array || is_ref_comp)) { + H5Tclose(mtype_id); + H5Tclose(ftype_id); + H5Aclose(attr_id); + continue; + } + + /* get name */ + if(H5Aget_name(attr_id, 255, name) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_name failed"); + + /* get the dataspace handle */ + if((space_id = H5Aget_space(attr_id)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_space failed"); + + /* get dimensions */ + if((rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed"); + + + /*------------------------------------------------------------------------- + * elements + *------------------------------------------------------------------------- + */ + nelmts = 1; + for(j = 0; j < (unsigned)rank; j++) + nelmts *= dims[j]; + + if (is_ref_array) { + unsigned array_rank = 0; + hsize_t array_size = 1; + hsize_t array_dims[H5S_MAX_RANK]; + hid_t base_type; + + base_type = H5Tget_super(ftype_id); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + + array_rank = (unsigned)H5Tget_array_ndims(mtype_id); + H5Tget_array_dims2(mtype_id, array_dims); + for(j = 0; j <array_rank; j++) + array_size *= array_dims[j]; + nelmts *= array_size; + } + + if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Acreate2 failed"); + + if (nelmts>0) { + /* handle object references */ + if((is_ref || is_ref_array) && (H5R_OBJ_REF_BUF_SIZE==msize)) { + buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize)); + if(buf == NULL) { + printf("cannot read into memory\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + if(H5Aread(attr_id, mtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); + + refbuf = (hobj_ref_t *)HDcalloc((unsigned)nelmts, msize); + if(refbuf == NULL) { + printf("cannot allocate memory\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); + } /* end if */ + + for(i = 0; i < (unsigned)nelmts; i++) { + if (update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt)<0) + continue; + if(options->verbose) + printf("object <%s> reference created to <%s>\n", name, refname); + } /* i */ + } /* H5T_STD_REF_OBJ */ + /* handle region references */ + else if((is_ref || is_ref_array) && (H5R_DSET_REG_REF_BUF_SIZE == msize)) { + buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize)); + + if(buf == NULL) { + printf( "cannot read into memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + if(H5Aread(attr_id, mtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */ + if(refbuf == NULL) { + printf( "cannot allocate memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); + } /* end if */ + + for(i = 0; i < (unsigned)nelmts; i++) { + if (update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt)<0) + continue; + if(options->verbose) + printf("object <%s> region reference created to <%s>\n", name, refname); + } + } /* H5T_STD_REF_DSETREG */ + else if (is_ref_vlen) { + /* handle VLEN of references */ + + buf = (hvl_t *)HDmalloc((unsigned)(nelmts * sizeof(hvl_t))); + refbuf = buf; /* reuse the read buffer for write */ + + if(buf == NULL) { + printf( "cannot read into memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + + if(H5Aread(attr_id, mtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); + + if (H5R_OBJ_REF_BUF_SIZE==msize) { + hobj_ref_t ref_out; + + for (i=0; i<(unsigned)nelmts; i++) { + hobj_ref_t *ptr = (hobj_ref_t *)((hvl_t *)buf)[i].p; + + for (j=0; j<((hvl_t *)buf)[i].len; j++ ) { + if (update_ref_value(attr_id, H5R_OBJECT, &(ptr[j]), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(&(ptr[j]), &ref_out, msize); + } + } /* for (i=0; i<nelems; i++) */ + } else if (H5R_DSET_REG_REF_BUF_SIZE == msize) { + hdset_reg_ref_t ref_out; + + for (i=0; i<(unsigned)nelmts; i++) { + hdset_reg_ref_t *ptr = (hdset_reg_ref_t *)((hvl_t *)buf)[i].p; + + for (j=0; j<((hvl_t *)buf)[i].len; j++ ) { + if (update_ref_value(attr_id, H5R_DATASET_REGION, &(ptr[j]), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(&(ptr[j]), &ref_out, msize); + } + } /* for (i=0; i<nelems; i++) */ + } + } /* else if (is_ref_vlen) */ + else if (is_ref_comp) { + /* handle ref fields in a compound */ + + buf = HDmalloc((unsigned)(nelmts * msize)); + refbuf = buf; /* reuse the read buffer for write */ + + if(buf == NULL) + { + printf( "cannot read into memory\n" ); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); + } /* end if */ + + if(H5Aread(attr_id, mtype_id, buf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); + + for (i=0; i<(unsigned)nelmts; i++) { + for (j=0; j<(unsigned)ref_comp_field_n; j++) { + if (ref_comp_size[j] == H5R_OBJ_REF_BUF_SIZE) { + size_t idx = (i * msize) + H5Tget_member_offset(mtype_id, ref_comp_index[j]); + hobj_ref_t ref_out; + + if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]); + } /* if */ + else if (ref_comp_size[j] == H5R_DSET_REG_REF_BUF_SIZE) { + size_t idx = i * msize + H5Tget_member_offset(mtype_id, ref_comp_index[j]); + hdset_reg_ref_t ref_out; + + if (update_ref_value(attr_id, H5R_DATASET_REGION, (hdset_reg_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]); + } /* else if */ + } /* j */ + } /* i */ + } /* else if (is_ref_comp) */ + + if(H5Awrite(attr_out, mtype_id, refbuf) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Awrite failed"); + + if (is_ref_vlen && buf) + H5Dvlen_reclaim (mtype_id, space_id, H5P_DEFAULT, buf); + } /* if (nelmts) */ + + if (refbuf == buf) + refbuf = NULL; /* set it to NULL to avoid double free since buf and refbuf are the same. */ + + if(buf) { + HDfree(buf); + buf = NULL; + } + + if(refbuf) { + HDfree(refbuf); + refbuf = NULL; + } + + if (ref_comp_index) { + HDfree(ref_comp_index); + ref_comp_index = NULL; + } + + if (ref_comp_size) { + HDfree(ref_comp_size); + ref_comp_size = NULL; + } + + if(H5Aclose(attr_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Tclose(ftype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if(H5Tclose(mtype_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + if(H5Sclose(space_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if(H5Aclose(attr_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); + } /* for(u = 0; u < (unsigned)oinfo.num_attrs; u++) */ + + return ret_value; + +done: + if(refbuf) + HDfree(refbuf); + if(buf) + HDfree(buf); + + if (ref_comp_index) + HDfree(ref_comp_index); + + if (ref_comp_size) + HDfree(ref_comp_size); + + H5E_BEGIN_TRY { + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + } H5E_END_TRY; + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: MapIdToName + * + * Purpose: map a ID from a reference to a dataset name + * + *------------------------------------------------------------------------- + */ +static const char* +MapIdToName(hid_t refobj_id, trav_table_t *travt) +{ + unsigned int u; + const char* ret = NULL; + + /* linear search */ + for(u = 0; u < travt->nobjs; u++) { + if(travt->objs[u].type == (h5trav_type_t)H5O_TYPE_DATASET || + travt->objs[u].type == (h5trav_type_t)H5O_TYPE_GROUP || + travt->objs[u].type == (h5trav_type_t)H5O_TYPE_NAMED_DATATYPE) { + H5O_info_t ref_oinfo; /* Stat for the refobj id */ + + /* obtain information to identify the referenced object uniquely */ + if(H5Oget_info(refobj_id, &ref_oinfo) < 0) + goto out; + + if(ref_oinfo.addr == travt->objs[u].objno) { + ret = travt->objs[u].name; + goto out; + } /* end if */ + } /* end if */ + } /* u */ + +out: + return ret; +} + +/*------------------------------------------------------------------------- + * Function: Update_Ref_value + * + * Purpose: Update a reference value + * + * Programmer: xcao@hdfgroup.org 9/12/2011 + * + *------------------------------------------------------------------------- + */ +static herr_t update_ref_value(hid_t obj_id, H5R_type_t ref_type, void *ref_in, + hid_t fid_out, void *ref_out, trav_table_t *travt) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + const char* ref_obj_name; + hid_t space_id=-1, ref_obj_id=-1; + + ref_obj_id = H5Rdereference2(obj_id, H5P_DEFAULT, ref_type, ref_in); + if (ref_obj_id < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rdereference2 failed"); + + ref_obj_name = MapIdToName(ref_obj_id, travt); + if (ref_obj_name == NULL) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "MapIdToName failed"); + + if (ref_type == H5R_DATASET_REGION) { + space_id = H5Rget_region(obj_id, H5R_DATASET_REGION, ref_in); + if (space_id < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rget_region failed"); + } + + if(H5Rcreate(ref_out, fid_out, ref_obj_name, ref_type, space_id) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rcreate failed"); + +done: + H5E_BEGIN_TRY { + H5Sclose(space_id); + H5Oclose(ref_obj_id); + } H5E_END_TRY; + + return ret_value; +} + diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c new file mode 100644 index 0000000..6765c49 --- /dev/null +++ b/tools/src/h5repack/h5repack_verify.c @@ -0,0 +1,675 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/* number of members in an array */ +#ifndef NELMTS +# define NELMTS(X) (sizeof(X)/sizeof(X[0])) +#endif + +static int verify_layout(hid_t pid, pack_info_t *obj); +static int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter); + + +/*------------------------------------------------------------------------- + * Function: h5repack_verify + * + * Purpose: verify if filters and layout in the input file match the output file + * + * Return: + * 1 match + * 0 do not match + * -1 error + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 19, 2003 + * Modified: December, 19, 2007 (exactly 4 years later :-) ) + * Separate into 3 cases + * 1) no filter input, get all datasets and compare DCPLs. TO DO + * 2) filter input on selected datasets, get each one trough OBJ and match + * 3) filter input on all datasets, get all objects and match + * + *------------------------------------------------------------------------- + */ + +int +h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t fidin = -1; /* file ID for input file*/ + hid_t fidout = -1; /* file ID for output file*/ + hid_t did = -1; /* dataset ID */ + hid_t pid = -1; /* dataset creation property list ID */ + hid_t sid = -1; /* space ID */ + hid_t tid = -1; /* type ID */ + unsigned int i; + trav_table_t *travt = NULL; + int ok = 1; + hid_t fcpl_in = -1; /* file creation property for input file */ + hid_t fcpl_out = -1; /* file creation property for output file */ + H5F_file_space_type_t in_strat, out_strat; /* file space handling strategy for in/output file */ + hsize_t in_thresh, out_thresh; /* free space section threshold for in/output file */ + + /* open the output file */ + if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + return -1; + + for(i = 0; i < options->op_tbl->nelems; i++) + { + char* name = options->op_tbl->objs[i].path; + pack_info_t *obj = &options->op_tbl->objs[i]; + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if((sid = H5Dget_space(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); + if((pid = H5Dget_create_plist(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + if((tid = H5Dget_type(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(verify_filters(pid, tid, obj->nfilters, obj->filter) <= 0) + ok = 0; + + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if((obj->layout != -1) && (verify_layout(pid, obj) == 0)) + ok = 0; + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(pid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Sclose(sid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if (H5Dclose(did) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + if (H5Tclose(tid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + + } + + + /*------------------------------------------------------------------------- + * check for the "all" objects option + *------------------------------------------------------------------------- + */ + + if(options->all_filter == 1 || options->all_layout == 1) + { + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if(h5trav_gettable(fidout, travt) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed"); + + for(i = 0; i < travt->nobjs; i++) + { + char *name = travt->objs[i].name; + + if(travt->objs[i].type == H5TRAV_TYPE_DATASET) + { + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if((sid = H5Dget_space(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); + if((pid = H5Dget_create_plist(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + if((tid = H5Dget_type(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(options->all_filter == 1) + { + if(verify_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0) + ok = 0; + } + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if(options->all_layout == 1) + { + pack_info_t pack; + + init_packobject(&pack); + pack.layout = options->layout_g; + pack.chunk = options->chunk_g; + if(verify_layout(pid, &pack) == 0) + ok = 0; + } + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(pid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Sclose(sid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); + if (H5Dclose(did) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + if (H5Tclose(tid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); + } /* if */ + + } /* i */ + + /* free table */ + trav_table_free(travt); + } + + /*------------------------------------------------------------------------- + * Verify that file space strategy and free space threshold + * are set as expected + *------------------------------------------------------------------------- + */ + + /* open the input file */ + if((fidin = H5Fopen(in_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fopen failed"); + + /* Get file creation property list for input file */ + if((fcpl_in = H5Fget_create_plist(fidin)) < 0) { + error_msg("failed to retrieve file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + + /* Get file space management info for input file */ + if(H5Pget_file_space(fcpl_in, &in_strat, &in_thresh) < 0) { + error_msg("failed to retrieve file space strategy & threshold\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); + } + + /* Output file is already opened */ + /* Get file creation property list for output file */ + if((fcpl_out = H5Fget_create_plist(fidout)) < 0) { + error_msg("failed to retrieve file creation property list\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + + /* Get file space management info for output file */ + if(H5Pget_file_space(fcpl_out, &out_strat, &out_thresh) < 0) { + error_msg("failed to retrieve file space strategy & threshold\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); + } + + /* + * If the strategy option is not set, + * file space handling strategy should be the same for both + * input & output files. + * If the strategy option is set, + * the output file's file space handling strategy should be the same + * as what is set via the strategy option + */ + if(!options->fs_strategy && out_strat != in_strat) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + + } + else if(options->fs_strategy && out_strat!= options->fs_strategy) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + + /* + * If the threshold option is not set, + * the free space section threshold should be the same for both + * input & output files. + * If the threshold option is set, + * the output file's free space section threshold should be the same + * as what is set via the threshold option. + */ + if(!options->fs_threshold && out_thresh != in_thresh) { + error_msg("free space threshold not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + } + else if(options->fs_threshold && out_thresh != options->fs_threshold) { + error_msg("free space threshold not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + } + + /* Closing */ + if (H5Pclose(fcpl_in) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Pclose(fcpl_out) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Fclose(fidin) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed"); + if (H5Fclose(fidout) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed"); + + return ok; + +done: + H5E_BEGIN_TRY { + H5Pclose(fcpl_in); + H5Pclose(fcpl_out); + H5Pclose(pid); + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fidin); + H5Fclose(fidout); + if (travt) + trav_table_free(travt); + } H5E_END_TRY; + + return ret_value; +} /* h5repack_verify() */ + + + +/*------------------------------------------------------------------------- + * Function: verify_layout + * + * Purpose: verify which layout is present in the property list DCPL_ID + * + * H5D_COMPACT = 0 + * H5D_CONTIGUOUS = 1 + * H5D_CHUNKED = 2 + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ + +int verify_layout(hid_t pid, + pack_info_t *obj) +{ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; /* layout */ + int nfilters; /* number of filters */ + int rank; /* rank */ + int i; /* index */ + + /* check if we have filters in the input object */ + if ((nfilters = H5Pget_nfilters(pid)) < 0) + return -1; + + /* a non chunked layout was requested on a filtered object */ + if (nfilters && obj->layout!=H5D_CHUNKED) + return 0; + + /* get layout */ + if ((layout = H5Pget_layout(pid)) < 0) + return -1; + + if (obj->layout != layout) + return 0; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(pid,NELMTS(chsize),chsize/*out*/)) < 0) + return -1; + if (obj->chunk.rank != rank) + return 0; + for ( i=0; i<rank; i++) + if (chsize[i] != obj->chunk.chunk_lengths[i]) + return 0; + } + + return 1; +} + +/*------------------------------------------------------------------------- + * Function: h5repack_cmp_pl + * + * Purpose: compare 2 files for identical property lists of all objects + * + * Return: 1=identical, 0=not identical, -1=error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 31, 2003 + * + *------------------------------------------------------------------------- + */ + +int h5repack_cmp_pl(const char *fname1, + const char *fname2) +{ + int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ + hid_t fid1=-1; /* file ID */ + hid_t fid2=-1; /* file ID */ + hid_t dset1=-1; /* dataset ID */ + hid_t dset2=-1; /* dataset ID */ + hid_t gid=-1; /* group ID */ + hid_t dcpl1=-1; /* dataset creation property list ID */ + hid_t dcpl2=-1; /* dataset creation property list ID */ + hid_t gcplid=-1; /* group creation property list */ + unsigned crt_order_flag1; /* group creation order flag */ + unsigned crt_order_flag2; /* group creation order flag */ + trav_table_t *trav=NULL; + int ret=1; + unsigned int i; + + /*------------------------------------------------------------------------- + * open the files + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY + { + + /* Open the files */ + if ((fid1 = H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg("<%s>: %s\n", fname1, H5FOPENERROR ); + return -1; + } + if ((fid2 = H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg("<%s>: %s\n", fname2, H5FOPENERROR ); + H5Fclose(fid1); + return -1; + } + /* enable error reporting */ + } H5E_END_TRY; + + /*------------------------------------------------------------------------- + * get file table list of objects + *------------------------------------------------------------------------- + */ + trav_table_init(&trav); + if(h5trav_gettable(fid1, trav) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed"); + + /*------------------------------------------------------------------------- + * traverse the suppplied object list + *------------------------------------------------------------------------- + */ + for(i = 0; i < trav->nobjs; i++) + { + + if(trav->objs[i].type == H5TRAV_TYPE_GROUP) + { + + if ((gid = H5Gopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + if ((gcplid = H5Gget_create_plist(gid)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed"); + if (H5Pget_link_creation_order(gcplid, &crt_order_flag1) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed"); + if (H5Pclose(gcplid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Gclose(gid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + + if ((gid = H5Gopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); + if ((gcplid = H5Gget_create_plist(gid)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed"); + if (H5Pget_link_creation_order(gcplid, &crt_order_flag2) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed"); + if (H5Pclose(gcplid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if (H5Gclose(gid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); + + if (crt_order_flag1 != crt_order_flag2) { + error_msg("property lists for <%s> are different\n",trav->objs[i].name); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "property lists failed"); + } + + } + + + + else if(trav->objs[i].type == H5TRAV_TYPE_DATASET) + { + if((dset1 = H5Dopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if((dset2 = H5Dopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); + if((dcpl1 = H5Dget_create_plist(dset1)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + if((dcpl2 = H5Dget_create_plist(dset2)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); + + /*------------------------------------------------------------------------- + * compare the property lists + *------------------------------------------------------------------------- + */ + if((ret = H5Pequal(dcpl1, dcpl2)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pequal failed"); + + if(ret == 0) { + error_msg("property lists for <%s> are different\n",trav->objs[i].name); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "property lists failed"); + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(dcpl1) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if(H5Pclose(dcpl2) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + if(H5Dclose(dset1) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + if(H5Dclose(dset2) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); + } /*if*/ + } /*i*/ + + /*------------------------------------------------------------------------- + * free + *------------------------------------------------------------------------- + */ + + trav_table_free(trav); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + H5Fclose(fid1); + H5Fclose(fid2); + + return ret; + +/*------------------------------------------------------------------------- +* error +*------------------------------------------------------------------------- +*/ +done: + H5E_BEGIN_TRY + { + H5Pclose(dcpl1); + H5Pclose(dcpl2); + H5Dclose(dset1); + H5Dclose(dset2); + H5Fclose(fid1); + H5Fclose(fid2); + H5Pclose(gcplid); + H5Gclose(gid); + trav_table_free(trav); + } H5E_END_TRY; + + return ret_value; +} + + +/*------------------------------------------------------------------------- + * Function: verify_filters + * + * Purpose: verify if all requested filters in the array FILTER obtained + * from user input are present in the property list PID obtained from + * the output file + * + * Return: + * 1 match + * 0 do not match + * -1 error + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 21, 2007 + * + *------------------------------------------------------------------------- + */ + +static +int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter) +{ + int nfilters_dcpl; /* number of filters in DCPL*/ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + size_t size; /* type size */ + int i; /* index */ + unsigned j; /* index */ + + /* get information about filters */ + if((nfilters_dcpl = H5Pget_nfilters(pid)) < 0) + return -1; + + /* if we do not have filters and the requested filter is NONE, return 1 */ + if(!nfilters_dcpl && + nfilters == 1 && + filter[0].filtn == H5Z_FILTER_NONE ) + return 1; + + /* else the numbers of filters must match */ + if (nfilters_dcpl != nfilters ) + return 0; + + /*------------------------------------------------------------------------- + * build a list with DCPL filters + *------------------------------------------------------------------------- + */ + + for( i = 0; i < nfilters_dcpl; i++) + { + cd_nelmts = NELMTS(cd_values); + filtn = H5Pget_filter2(pid, (unsigned)i, &filt_flags, &cd_nelmts, + cd_values, sizeof(f_name), f_name, NULL); + + /* filter ID */ + if (filtn != filter[i].filtn) + return 0; + + /* compare client data values. some filters do return local values */ + switch (filtn) + { + + case H5Z_FILTER_NONE: + break; + + case H5Z_FILTER_SHUFFLE: + /* 1 private client value is returned by DCPL */ + if ( cd_nelmts != H5Z_SHUFFLE_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SHUFFLE_USER_NPARMS ) + return 0; + + /* get dataset's type size */ + if((size = H5Tget_size(tid)) <= 0) + return -1; + + /* the private client value holds the dataset's type size */ + if ( size != cd_values[0] ) + return 0; + + break; + + case H5Z_FILTER_SZIP: + /* 4 private client values are returned by DCPL */ + if ( cd_nelmts != H5Z_SZIP_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SZIP_USER_NPARMS ) + return 0; + + /* "User" parameter for pixels-per-block (index 1) */ + if ( cd_values[H5Z_SZIP_PARM_PPB] != filter[i].cd_values[H5Z_SZIP_PARM_PPB] ) + return 0; + + break; + + case H5Z_FILTER_NBIT: + /* only client data values number of values checked */ + if ( H5Z_NBIT_USER_NPARMS != filter[i].cd_nelmts) + return 0; + break; + + case H5Z_FILTER_SCALEOFFSET: + /* only client data values checked */ + for( j = 0; j < H5Z_SCALEOFFSET_USER_NPARMS; j++) + if (cd_values[j] != filter[i].cd_values[j]) + return 0; + break; + + /* for these filters values must match, no local values set in DCPL */ + case H5Z_FILTER_FLETCHER32: + case H5Z_FILTER_DEFLATE: + + if ( cd_nelmts != filter[i].cd_nelmts) + return 0; + + for( j = 0; j < cd_nelmts; j++) + if (cd_values[j] != filter[i].cd_values[j]) + return 0; + + break; + + default: + if ( cd_nelmts != filter[i].cd_nelmts) + return 0; + + for( j = 0; j < cd_nelmts; j++) + if (cd_values[j] != filter[i].cd_values[j]) + return 0; + break; + + } /* switch */ + } + + return 1; +} + |