From d8531b84827b78bc481ccdac77693fb9247ba979 Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Tue, 30 Dec 2003 17:59:33 -0500 Subject: [svn-r7997] Purpose: h5repack new features Description: added support for layout options Solution: Platforms tested: linux solaris AIX Misc. update: --- tools/h5repack/Makefile.in | 6 +- tools/h5repack/h5repack.c | 79 ++++++----- tools/h5repack/h5repack.h | 41 ++++-- tools/h5repack/h5repack_copy.c | 82 +++--------- tools/h5repack/h5repack_filters.c | 57 ++++++++ tools/h5repack/h5repack_layout.c | 121 +++++++++++++++++ tools/h5repack/h5repack_main.c | 2 +- tools/h5repack/h5repack_opttable.c | 46 ++++--- tools/h5repack/h5repack_parse.c | 184 ++++++++++++++++---------- tools/h5repack/h5repack_verify.c | 95 +++++++++++--- tools/h5repack/testh5repack_main.c | 261 +++++++++++++++++++++++++++++++++++-- 11 files changed, 744 insertions(+), 230 deletions(-) create mode 100644 tools/h5repack/h5repack_layout.c diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in index 5b3cbc0..83bbee1 100644 --- a/tools/h5repack/Makefile.in +++ b/tools/h5repack/Makefile.in @@ -50,10 +50,10 @@ MOSTLYCLEAN=*.h5 ## Source and object files for programs... ## -PROG_SRC=h5repack.c h5repack_copy.c h5repack_filters.c h5repack_refs.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c h5repack_verify.c testh5repack_attr.c testh5repack_dset.c testh5repack_filters.c testh5repack_main.c testh5repack_make.c +PROG_SRC=h5repack.c h5repack_copy.c h5repack_filters.c h5repack_refs.c h5repack_layout.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c h5repack_verify.c testh5repack_attr.c testh5repack_dset.c testh5repack_filters.c testh5repack_main.c testh5repack_make.c PROG_OBJ=$(PROG_SRC:.c=.lo) -OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo -TEST_OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo testh5repack_attr.lo testh5repack_dset.lo testh5repack_filters.lo testh5repack_main.lo testh5repack_make.lo +OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_layout.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo +TEST_OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_layout.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo testh5repack_attr.lo testh5repack_dset.lo testh5repack_filters.lo testh5repack_main.lo testh5repack_make.lo PRIVATE_HDR= diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 6ba20bd..e4a83a0 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -142,10 +142,9 @@ int h5repack_addfilter(const char* str, /*------------------------------------------------------------------------- - * Function: h5repack_addchunk + * Function: h5repack_addlayout * - * Purpose: add a chunk -c option to table - * Example: -c "*:2x2" , STR = "*:2x2" + * Purpose: add a layout option * * Return: 0, ok, -1, fail * @@ -153,36 +152,43 @@ int h5repack_addfilter(const char* str, */ -int h5repack_addchunk(const char* str, - pack_opt_t *options) +int h5repack_addlayout(const char* str, + pack_opt_t *options) { obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ int n_objs; /*number of objects in the current -t or -c option entry */ - hsize_t chunk_lengths[MAX_VAR_DIMS]; /* chunk lengths along each dimension */ - int chunk_rank; /*global rank for chunks */ int j; + pack_info_t pack; /*info about layout to extract from parse */ - if (options->all_chunk==1){ - printf("Error: Invalid chunking input: '*' is present with other objects <%s>\n",str); + if (options->all_layout==1){ + printf("Error: Invalid layout input: all option \ + is present with other objects <%s>\n",str); return -1; } - /* parse the -c option */ - obj_list=parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank,options); + /* parse the layout option */ + obj_list=parse_layout(str,&n_objs,&pack,options); if (obj_list==NULL) return -1; - if (options->all_chunk==1) + /* set global layout option */ + if (options->all_layout==1 ) { - /* if we are chunking all set the global chunking type */ - options->chunk_g.rank=chunk_rank; - for (j = 0; j < chunk_rank; j++) - options->chunk_g.chunk_lengths[j] = chunk_lengths[j]; + options->layout_g=pack.layout; + if (pack.layout==H5D_CHUNKED) { + /* if we are chunking all set the global chunking type */ + options->chunk_g.rank=pack.chunk.rank; + for (j = 0; j < pack.chunk.rank; j++) + options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; + } } - if (options->all_chunk==0) - options_add_chunk(obj_list,n_objs,chunk_lengths,chunk_rank,options->op_tbl); + if (options->all_layout==0) + options_add_layout(obj_list, + n_objs, + &pack, + options->op_tbl); free(obj_list); return 0; @@ -210,18 +216,21 @@ static int check_options(pack_opt_t *options) unsigned szip_pixels_per_block; /*------------------------------------------------------------------------- - * objects to chunk + * objects to layout *------------------------------------------------------------------------- */ if (options->verbose) { printf("\n"); - printf("Objects to chunk are...\n"); - if (options->all_chunk==1) { - printf("\tChunk all with dimension ["); - for ( j = 0; j < options->chunk_g.rank; j++) - printf("%d ",(int)options->chunk_g.chunk_lengths[j]); - printf("]\n"); + printf("Objects to modify are...\n"); + if (options->all_layout==1) { + printf("\tApply layout to all\n "); + if (H5D_CHUNKED==options->layout_g) { + printf("with dimension ["); + for ( j = 0; j < options->chunk_g.rank; j++) + printf("%d ",(int)options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } } }/* verbose */ @@ -247,7 +256,7 @@ static int check_options(pack_opt_t *options) } } - if (options->all_chunk==1 && has_ck){ + if (options->all_layout==1 && has_ck){ printf("Error: Invalid chunking input: all option\ is present with other objects\n"); return -1; @@ -283,30 +292,30 @@ static int check_options(pack_opt_t *options) for ( i = 0; i < options->op_tbl->nelems; i++) { - pack_info_t obj = options->op_tbl->objs[i]; - char* name = obj.path; - if (obj.filter.filtn>0) + pack_info_t pack = options->op_tbl->objs[i]; + char* name = pack.path; + if (pack.filter.filtn>0) { if (options->verbose) { printf("\t<%s> with %s filter", name, - get_sfilter(obj.filter.filtn)); + get_sfilter(pack.filter.filtn)); } has_cp=1; /*check for invalid combination of options */ - switch (obj.filter.filtn) + switch (pack.filter.filtn) { default: break; case H5Z_FILTER_SZIP: - szip_pixels_per_block=obj.filter.cd_values[0]; + szip_pixels_per_block=pack.filter.cd_values[0]; /* check szip parameters */ - if (check_szip(obj.chunk.rank, - obj.chunk.chunk_lengths, + if (check_szip(pack.chunk.rank, + pack.chunk.chunk_lengths, 0, /* do not test size */ szip_options_mask, szip_pixels_per_block)==0) @@ -422,7 +431,7 @@ void read_info(const char *filename, } comp_info[i-1]='\0'; /*cut the last " */ - if (h5repack_addchunk(comp_info,options)==-1){ + if (h5repack_addlayout(comp_info,options)==-1){ printf( "Could not add chunck option. Exiting\n"); exit(1); } diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index ae20cc4..cee7c1e 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -68,6 +68,7 @@ typedef struct { typedef struct { char path[MAX_NC_NAME]; /* name of object */ filter_info_t filter; /* filter information */ + H5D_layout_t layout; /* layout information */ chunk_info_t chunk; /* chunk information */ hid_t refobj_id; /* object ID, references */ } pack_info_t; @@ -88,10 +89,11 @@ typedef struct { /* all the above, ready to go to the hrepack call */ typedef struct { pack_opttbl_t *op_tbl; /*table with all -c and -f options */ - int all_chunk; /*chunk all objects */ - int all_filter; /*filter all objects */ + int all_layout; /*apply the layout to all objects */ + int all_filter; /*apply the filter to all objects */ filter_info_t filter_g; /*global filter INFO for the ALL case */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /*global layout information for the ALL case */ int verbose; /*verbose mode */ int threshold; /*minimum size to compress, in bytes */ } pack_opt_t; @@ -109,7 +111,7 @@ extern "C" { int h5repack (const char* infile, const char* outfile, pack_opt_t *options); int h5repack_addfilter (const char* str, pack_opt_t *options); -int h5repack_addchunk (const char* str, pack_opt_t *options); +int h5repack_addlayout (const char* str, pack_opt_t *options); int h5repack_init (pack_opt_t *options, int verbose); int h5repack_end (pack_opt_t *options); int h5repack_verify (const char *fname,pack_opt_t *options); @@ -171,10 +173,14 @@ void close_obj(H5G_obj_t obj_type, hid_t obj_id); *------------------------------------------------------------------------- */ +int filter_this(const char* name, + pack_opt_t *options, + pack_info_t *pack); /* info about object to filter */ + int apply_filter(hid_t dcpl_id, size_t size, /* size of datatype in bytes */ pack_opt_t *options, /* repack options */ - pack_info_t *obj); /* info about object to filter */ + pack_info_t *pack); /* info about object to filter */ int has_filter(hid_t dcpl_id, H5Z_filter_t filtnin); @@ -191,6 +197,21 @@ int check_szip(int rank, /* chunk rank */ unsigned szip_pixels_per_block); +/*------------------------------------------------------------------------- + * layout functions + *------------------------------------------------------------------------- + */ + +int has_layout(hid_t dcpl_id, + pack_info_t *obj); + +int layout_this(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *pack); /* info about object to apply layout */ + +int apply_layout(hid_t dcpl_id, + pack_opt_t *options, /* repack options */ + pack_info_t *pack); /* info about object */ /*------------------------------------------------------------------------- @@ -199,10 +220,9 @@ int check_szip(int rank, /* chunk rank */ */ int options_table_init( pack_opttbl_t **tbl ); int options_table_free( pack_opttbl_t *table ); -int options_add_chunk ( obj_list_t *obj_list, +int options_add_layout( obj_list_t *obj_list, int n_objs, - hsize_t *chunk_lengths, - int chunk_rank, + pack_info_t *pack, pack_opttbl_t *table ); int options_add_filter ( obj_list_t *obj_list, int n_objs, @@ -220,11 +240,12 @@ obj_list_t* parse_filter(const char *str, int *n_objs, filter_info_t *filt, pack_opt_t *options); -obj_list_t* parse_chunk (const char *str, + +obj_list_t* parse_layout(const char *str, int *n_objs, - hsize_t *chunk_lengths, - int *chunk_rank, + pack_info_t *pack, /* info about object */ pack_opt_t *options); + const char* get_sfilter (H5Z_filter_t filtn); int parse_number(char *str); diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 4d56ae6..569af82 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -19,11 +19,6 @@ #include "H5private.h" #include "h5repack.h" -static int filter_this(const char* name, - pack_opt_t *options, - pack_info_t *obj); /* info about object to filter */ - - /*------------------------------------------------------------------------- * Function: copy_file * @@ -246,8 +241,8 @@ int do_copy_file(hid_t fidin, if ( ! H5Tequal(mtype_id, H5T_STD_REF_OBJ) ) { - /* the information about the object to be filtered */ - pack_info_t filt_obj; + /* the information about the object to be filtered/"layouted" */ + pack_info_t pack; /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); @@ -264,15 +259,26 @@ int do_copy_file(hid_t fidin, } if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; + + /*------------------------------------------------------------------------- + * apply the layout; check first if the object is to be modified. + * if the layout could not be applied, continue + *------------------------------------------------------------------------- + */ + if (layout_this(travt->objs[i].name,options,&pack)) + { + if (apply_layout(dcpl_id,options,&pack)<0) + continue; + } /*------------------------------------------------------------------------- * apply the filter; check first if the object is to be filtered. * if the filter could not be applied, continue *------------------------------------------------------------------------- */ - if (filter_this(travt->objs[i].name,options,&filt_obj)) + if (filter_this(travt->objs[i].name,options,&pack)) { - if (apply_filter(dcpl_id,H5Tget_size(mtype_id),options,&filt_obj)<0) + if (apply_filter(dcpl_id,H5Tget_size(mtype_id),options,&pack)<0) continue; } @@ -603,61 +609,3 @@ error: return -1; } - - - -/*------------------------------------------------------------------------- - * Function: filter_this - * - * Purpose: find the object name NAME (got from the traverse list) - * in the repack options list; assign the filter information OBJ - * - * Return: 0 not found, 1 found - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * - * Date: December 19, 2003 - * - *------------------------------------------------------------------------- - */ - -static -int filter_this(const char* name, /* object name from traverse list */ - pack_opt_t *options, /* repack options */ - pack_info_t *obj) /* info about object to filter */ -{ - char *pdest; - int result; - int i; - - /* if we are applying to all objects just return true */ - if (options->all_filter) - { - /* assign the global filter and chunk info to the OBJ info */ - obj->filter=options->filter_g; - obj->chunk=options->chunk_g; - return 1; - } - - for ( i=0; iop_tbl->nelems; i++) - { - if (strcmp(options->op_tbl->objs[i].path,name)==0) - { - *obj=options->op_tbl->objs[i]; - return 1; - } - - pdest = strstr(name,options->op_tbl->objs[i].path); - result = (int)(pdest - name); - - /* found at position 1, meaning without '/' */ - if( pdest != NULL && result==1 ) - { - *obj=options->op_tbl->objs[i]; - return 1; - } - } - - return 0; -} - diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 09e2e75..5f7f641 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -16,6 +16,63 @@ #include "h5test.h" #include "h5repack.h" +/*------------------------------------------------------------------------- + * Function: filter_this + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the filter information OBJ + * + * Return: 0 not found, 1 found + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int filter_this(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj) /* info about object to filter */ +{ + char *pdest; + int result; + int i; + + /* if we are applying to all objects just return true */ + if (options->all_filter) + { + /* assign the global filter and chunk info to the OBJ info */ + obj->filter=options->filter_g; + obj->chunk=options->chunk_g; + return 1; + } + + for ( i=0; iop_tbl->nelems; i++) + { + if (options->op_tbl->objs[i].filter.filtn != -1 ) + { + if (strcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + + pdest = strstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + } + } + + return 0; +} + /*------------------------------------------------------------------------- diff --git a/tools/h5repack/h5repack_layout.c b/tools/h5repack/h5repack_layout.c new file mode 100644 index 0000000..0fff459 --- /dev/null +++ b/tools/h5repack/h5repack_layout.c @@ -0,0 +1,121 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + +/*------------------------------------------------------------------------- + * Function: layout_this + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the layout information OBJ + * + * Return: 0 not found, 1 found + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ + +int layout_this(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *pack /*OUT*/) /* object to apply layout */ +{ + char *pdest; + int result; + int i; + + /* if we are applying to all objects just return true */ + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + pack->layout=options->layout_g; + if (H5D_CHUNKED==options->layout_g) { /* set up chunk */ + pack->chunk.rank=options->chunk_g.rank; + for ( i=0; ichunk.rank; i++) + pack->chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + } + return 1; + } + + for ( i=0; iop_tbl->nelems; i++) + { + if (options->op_tbl->objs[i].layout != -1 ) + { + if (strcmp(options->op_tbl->objs[i].path,name)==0) + { + *pack=options->op_tbl->objs[i]; + return 1; + } + + pdest = strstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *pack=options->op_tbl->objs[i]; + return 1; + } + } + } + + return 0; +} +/*------------------------------------------------------------------------- + * Function: apply_layout + * + * Purpose: apply a layout to the property list. Valid values for layout are: + * + * H5D_COMPACT + * Store raw data in the dataset object header in file. + * This should only be used for very small amounts of raw data. + * H5D_CONTIGUOUS + * Store raw data separately from the object header in one large chunk + * in the file. + * H5D_CHUNKED + * Store raw data separately from the object header as chunks of data in + * separate locations in the file. + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ + +int apply_layout(hid_t dcpl_id, + pack_opt_t *options, /* repack options */ + pack_info_t *obj) /* info about object */ +{ + + + if (H5Pset_layout(dcpl_id, obj->layout)<0) + return -1; + + if (H5D_CHUNKED==obj->layout) { /* set up chunk */ + if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0) + return -1; + } + + + return 0; +} + diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index b7dd9bf..bfa8366 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -59,7 +59,7 @@ int main(int argc, char **argv) else if (strcmp(argv[i], "-c") == 0) { /* parse the -c option */ - h5repack_addchunk(argv[i+1],&options); + h5repack_addlayout(argv[i+1],&options); /* jump to next */ ++i; diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 0571656..a18c508 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -52,6 +52,7 @@ int options_table_init( pack_opttbl_t **tbl ) table->objs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; table->objs[i].refobj_id = -1; + table->objs[i].layout = -1; } *tbl = table; @@ -76,9 +77,9 @@ int options_table_free( pack_opttbl_t *table ) } /*------------------------------------------------------------------------- - * Function: options_add_chunk + * Function: options_add_layout * - * Purpose: add a chunking -c option to the option list + * Purpose: add a layout option to the option list * * Return: 0, ok, -1, fail * @@ -86,11 +87,10 @@ int options_table_free( pack_opttbl_t *table ) */ -int options_add_chunk( obj_list_t *obj_list, - int n_objs, - hsize_t *chunk_lengths, - int chunk_rank, - pack_opttbl_t *table ) +int options_add_layout( obj_list_t *obj_list, + int n_objs, + pack_info_t *pack, + pack_opttbl_t *table ) { int i, j, k, I, added=0, found=0; @@ -108,8 +108,10 @@ int options_add_chunk( obj_list_t *obj_list, table->objs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; table->objs[i].refobj_id = -1; + table->objs[i].layout = -1; } } + /* search if this object is already in the table; "path" is the key */ if (table->nelems>0) @@ -129,12 +131,15 @@ int options_add_chunk( obj_list_t *obj_list, printf("Input Error: chunk information already inserted for <%s>\n",obj_list[j].obj); exit(1); } - /* insert the chunk info */ + /* insert the layout info */ else { - table->objs[i].chunk.rank = chunk_rank; - for (k = 0; k < chunk_rank; k++) - table->objs[i].chunk.chunk_lengths[k] = chunk_lengths[k]; + table->objs[i].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[i].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[i].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } found=1; break; } @@ -147,9 +152,12 @@ int options_add_chunk( obj_list_t *obj_list, I = table->nelems + added; added++; strcpy(table->objs[I].path,obj_list[j].obj); - table->objs[I].chunk.rank = chunk_rank; - for (k = 0; k < chunk_rank; k++) - table->objs[I].chunk.chunk_lengths[k] = chunk_lengths[k]; + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } } } /* j */ } @@ -163,9 +171,12 @@ int options_add_chunk( obj_list_t *obj_list, I = table->nelems + added; added++; strcpy(table->objs[I].path,obj_list[j].obj); - table->objs[I].chunk.rank = chunk_rank; - for (k = 0; k < chunk_rank; k++) - table->objs[I].chunk.chunk_lengths[k] = chunk_lengths[k]; + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } } } @@ -208,6 +219,7 @@ int options_add_filter(obj_list_t *obj_list, table->objs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; table->objs[i].refobj_id = -1; + table->objs[i].layout = -1; } } diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index 9f331b2..aa6ff82 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -26,6 +26,13 @@ * * Return: a list of names, the number of names and its compression type * + * can be: + * GZIP, to apply the HDF5 GZIP filter (GZIP compression) + * SZIP, to apply the HDF5 SZIP filter (SZIP compression) + * SHUF, to apply the HDF5 shuffle filter + * FLET, to apply the HDF5 checksum filter + * NONE, to remove the filter + * * Examples: * "GZIP 6" * "A,B:NONE" @@ -47,7 +54,7 @@ obj_list_t* parse_filter(const char *str, char c; size_t len=strlen(str); int j, m, n, k, end_obj=-1, no_param=0; - char obj[MAX_NC_NAME]; + char sobj[MAX_NC_NAME]; char scomp[10]; char stype[5]; obj_list_t* obj_list=NULL; @@ -88,12 +95,12 @@ obj_list_t* parse_filter(const char *str, for ( j=0, k=0, n=0; jall_chunk=1; + options->all_layout=1; } n++; @@ -337,12 +348,12 @@ obj_list_t* parse_chunk(const char *str, for ( j=0, k=0, n=0; j\n",str); + printf("Parse layout error: No characters after : in <%s>\n",str); exit(1); } - /* get chunk info */ - k=0; - for ( i=end_obj+1, c_index=0; i\n",str); - exit(1); - } - - if ( c=='x' || i==len-1) + if (n==5) { - if ( c=='x') { - sdim[k-1]='\0'; - k=0; - chunk_lengths[c_index]=atoi(sdim); - if (chunk_lengths[c_index]==0) { - if (obj_list) free(obj_list); - printf("Input Error: Invalid chunking in <%s>\n",str); - exit(1); - } - c_index++; + slayout[n]='\0'; /*cut string */ + if (strcmp(slayout,"COMPA")==0) + pack->layout=H5D_COMPACT; + else if (strcmp(slayout,"CONTI")==0) + pack->layout=H5D_CONTIGUOUS; + else if (strcmp(slayout,"CHUNK")==0) + pack->layout=H5D_CHUNKED; + else { + printf("Parse layout error: Not a valid layout in <%s>\n",str); } - else if (i==len-1) { /*no more parameters */ - sdim[k]='\0'; - k=0; - if (strcmp(sdim,"NONE")==0) - { - *chunk_rank=-2; + } + else + { + c = str[j]; + slayout[n]=c; + } + } /* j */ + + + if ( pack->layout=H5D_CHUNKED ) + { + +/*------------------------------------------------------------------------- + * get chunk info + *------------------------------------------------------------------------- + */ + k=0; + for ( i=j, c_index=0; i Not a valid character in <%s>\n", + sdim,str); + exit(1); } - else + + if ( c=='x' || i==len-1) { - chunk_lengths[c_index]=atoi(sdim); - if (chunk_lengths[c_index]==0){ - if (obj_list) free(obj_list); - printf("Input Error: Invalid chunking in <%s>\n",str); - exit(1); + if ( c=='x') { + sdim[k-1]='\0'; + k=0; + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0) { + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Conversion to number in <%s>\n", + sdim,str); + exit(1); + } + c_index++; } - *chunk_rank=c_index+1; - } - } /*if */ - } /*if c=='x' || i==len-1 */ - } /*i*/ + else if (i==len-1) { /*no more parameters */ + sdim[k]='\0'; + k=0; + if (strcmp(sdim,"NONE")==0) + { + pack->chunk.rank=-2; + } + else + { + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0){ + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Conversion to number in <%s>\n", + sdim,str); + exit(1); + } + pack->chunk.rank=c_index+1; + } + } /*if */ + } /*if c=='x' || i==len-1 */ + } /*i*/ + + + } /*H5D_CHUNKED*/ + return obj_list; } diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index f3c1583..4486477 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -43,9 +43,6 @@ int has_filter(hid_t dcpl_id, size_t cd_nelmts; /* filter client number of values */ char f_name[256]; /* filter name */ int have=0; /* flag, filter is present */ - hsize_t chsize[64]; /* chunk size in elements */ - H5D_layout_t layout; /* layout */ - int rank; /* rank */ int i; /* index */ /* if no information about the input filter is requested return exit */ @@ -55,18 +52,6 @@ int has_filter(hid_t dcpl_id, /* get information about filters */ if ((nfilters = H5Pget_nfilters(dcpl_id))<0) return -1; - -/* - H5D_COMPACT = 0 - H5D_CONTIGUOUS = 1 - H5D_CHUNKED = 2 - */ - layout = H5Pget_layout(dcpl_id); - if (layout==H5D_CHUNKED) - { - if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) - return -1; - } for (i=0; ilayout != layout) + return 0; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + if (obj->chunk.rank != rank) + return 0; + for ( i=0; ichunk.chunk_lengths[i]) + return 0; + } + + + + return 1; +} + /*------------------------------------------------------------------------- * Function: h5repack_verify * * Purpose: verify if the filters specified in the options list are - * present on the output file + * present on the OUTPUT file * * Return: 1=filter present, 0=filter not present, -1=error * @@ -142,6 +177,13 @@ int h5repack_verify(const char *fname, ret=0; /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if (has_layout(dcpl_id,&obj)==0) + ret=0; + +/*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ @@ -160,7 +202,7 @@ int h5repack_verify(const char *fname, *------------------------------------------------------------------------- */ - if (options->all_filter==1) + if (options->all_filter==1 || options->all_layout==1) { /* init table */ @@ -193,8 +235,23 @@ int h5repack_verify(const char *fname, * filter check *------------------------------------------------------------------------- */ - if (has_filter(dcpl_id,options->filter_g.filtn)==0) - ret=0; + if (options->all_filter==1){ + if (has_filter(dcpl_id,options->filter_g.filtn)==0) + ret=0; + } + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if (options->all_layout==1){ + pack_info_t pack; + pack.layout=options->layout_g; + pack.chunk=options->chunk_g; + if (has_layout(dcpl_id,&pack)==0) + ret=0; + } + /*------------------------------------------------------------------------- * close diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 4385247..ab5da12 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -103,7 +103,6 @@ error: * Programmer: Pedro Vicente * September, 19, 2003 * - * Modifications: * *------------------------------------------------------------------------- */ @@ -145,7 +144,7 @@ test_filter_deflate(void) TEST_ERROR; if (h5repack_addfilter("dset1:GZIP 9",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("dset1:20x10",&pack_options)<0) + if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -165,7 +164,7 @@ test_filter_deflate(void) TEST_ERROR; if (h5repack_addfilter("GZIP 9",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("20x10",&pack_options)<0) + if (h5repack_addlayout("CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -207,7 +206,6 @@ error: * Programmer: Pedro Vicente * December, 19, 2003 * - * Modifications: * *------------------------------------------------------------------------- */ @@ -232,7 +230,7 @@ test_filter_szip(void) TEST_ERROR; if (h5repack_addfilter("dset2:SZIP 8",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("dset2:20x10",&pack_options)<0) + if (h5repack_addlayout("dset2:CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -254,7 +252,7 @@ test_filter_szip(void) TEST_ERROR; if (h5repack_addfilter("SZIP 8",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("20x10",&pack_options)<0) + if (h5repack_addlayout("CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -322,7 +320,7 @@ test_filter_shuffle(void) TEST_ERROR; if (h5repack_addfilter("dset1:SHUF",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("dset1:20x10",&pack_options)<0) + if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -342,7 +340,7 @@ test_filter_shuffle(void) TEST_ERROR; if (h5repack_addfilter("SHUF",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("20x10",&pack_options)<0) + if (h5repack_addlayout("CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -384,8 +382,6 @@ error: * Programmer: Pedro Vicente * September, 19, 2003 * - * Modifications: - * *------------------------------------------------------------------------- */ static int @@ -409,7 +405,7 @@ test_filter_checksum(void) TEST_ERROR; if (h5repack_addfilter("dset1:FLET",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("dset1:20x10",&pack_options)<0) + if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -429,7 +425,7 @@ test_filter_checksum(void) TEST_ERROR; if (h5repack_addfilter("FLET",&pack_options)<0) TEST_ERROR; - if (h5repack_addchunk("20x10",&pack_options)<0) + if (h5repack_addlayout("CHUNK 20x10",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -455,6 +451,229 @@ error: } +/*------------------------------------------------------------------------- + * Function: test_layout_chunked + * + * Purpose: + * + * 1) test the CHUNK layout options + * 2) use the h5diff utility to compare the input and output file; + * it returns RET==0 if the objects have the same data + * 3) use API functions to verify the layout input on the output file + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * December 30, 2003 + * + *------------------------------------------------------------------------- + */ +static int +test_layout_chunked(void) +{ + pack_opt_t pack_options; + diff_opt_t diff_options; + memset(&diff_options, 0, sizeof (diff_opt_t)); + memset(&pack_options, 0, sizeof (pack_opt_t)); + + TESTING(" layout chunked"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("CHUNK 20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return 1; + + +} + + +/*------------------------------------------------------------------------- + * Function: test_layout_contiguous + * + * Purpose: + * + * 1) test the CONTI layout options + * 2) use the h5diff utility to compare the input and output file; + * it returns RET==0 if the objects have the same data + * 3) use API functions to verify the layout input on the output file + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * December 30, 2003 + * + *------------------------------------------------------------------------- + */ +static int +test_layout_contiguous(void) +{ + pack_opt_t pack_options; + diff_opt_t diff_options; + memset(&diff_options, 0, sizeof (diff_opt_t)); + memset(&pack_options, 0, sizeof (pack_opt_t)); + + TESTING(" layout contiguous"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return 1; + + +} + + +/*------------------------------------------------------------------------- + * Function: test_layout_compact + * + * Purpose: + * + * 1) test the COMPA layout options + * 2) use the h5diff utility to compare the input and output file; + * it returns RET==0 if the objects have the same data + * 3) use API functions to verify the layout input on the output file + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * December 30, 2003 + * + *------------------------------------------------------------------------- + */ +static int +test_layout_compact(void) +{ + pack_opt_t pack_options; + diff_opt_t diff_options; + memset(&diff_options, 0, sizeof (diff_opt_t)); + memset(&pack_options, 0, sizeof (pack_opt_t)); + + TESTING(" layout compact"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return 1; + + +} + + + /*------------------------------------------------------------------------- * Function: main @@ -483,6 +702,8 @@ int main (void) if (make_testfiles()<0) goto error; +#if 1 + /* test a copy with no filters */ nerrors += test_copy(); @@ -498,6 +719,22 @@ int main (void) /* test a copy with the checksum filter */ nerrors += test_filter_checksum(); + /* test a copy with layout CHUNK options */ + nerrors += test_layout_chunked(); + +#endif + +#if 0 + + /* test a copy with layout CONTI options */ + nerrors += test_layout_contiguous(); + + /* test a copy with layout COMPA options */ + nerrors += test_layout_compact(); + +#endif + + /* check for errors */ if (nerrors) goto error; -- cgit v0.12