diff options
author | cvs2svn <no_author@cvs2svn> | 2004-04-20 00:19:46 (GMT) |
---|---|---|
committer | cvs2svn <no_author@cvs2svn> | 2004-04-20 00:19:46 (GMT) |
commit | beb0079ae11d4932a1ce91545e49801ac206c75a (patch) | |
tree | 85b7e33258ef83d4dd9809f03a0cfb5477390720 /tools/h5repack | |
parent | c39de522e9559c382b7e705e7cb68fde263837b2 (diff) | |
download | hdf5-beb0079ae11d4932a1ce91545e49801ac206c75a.zip hdf5-beb0079ae11d4932a1ce91545e49801ac206c75a.tar.gz hdf5-beb0079ae11d4932a1ce91545e49801ac206c75a.tar.bz2 |
[svn-r8396] This commit was manufactured by cvs2svn to create branch 'hdf5_1_6'.
Diffstat (limited to 'tools/h5repack')
-rw-r--r-- | tools/h5repack/Makefile.in | 82 | ||||
-rw-r--r-- | tools/h5repack/h5repack.c | 497 | ||||
-rw-r--r-- | tools/h5repack/h5repack.h | 362 | ||||
-rw-r--r-- | tools/h5repack/h5repack.sh | 178 | ||||
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 620 | ||||
-rw-r--r-- | tools/h5repack/h5repack_filters.c | 541 | ||||
-rw-r--r-- | tools/h5repack/h5repack_list.c | 159 | ||||
-rw-r--r-- | tools/h5repack/h5repack_main.c | 148 | ||||
-rw-r--r-- | tools/h5repack/h5repack_opttable.c | 377 | ||||
-rw-r--r-- | tools/h5repack/h5repack_parse.c | 508 | ||||
-rw-r--r-- | tools/h5repack/h5repack_refs.c | 786 | ||||
-rw-r--r-- | tools/h5repack/h5repack_verify.c | 487 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_attr.c | 1032 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_dset.c | 826 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_filters.c | 166 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_layout.c | 108 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_main.c | 933 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_make.c | 400 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_util.c | 153 |
19 files changed, 8363 insertions, 0 deletions
diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in new file mode 100644 index 0000000..98f1fea --- /dev/null +++ b/tools/h5repack/Makefile.in @@ -0,0 +1,82 @@ +## +## Copyright by the Board of Trustees of the University of Illinois. +## All rights reserved. +## +## This file is part of HDF5. The full HDF5 copyright notice, including +## terms governing use, modification, and redistribution, is contained in +## the files COPYING and Copyright.html. COPYING can be found at the root +## of the source code distribution tree; Copyright.html can be found at the +## root level of an installed copy of the electronic HDF5 document set and +## is linked from the top-level documents page. It can also be found at +## http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have +## access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. +## +## HDF5 Library Makefile(.in) +## +top_srcdir=@top_srcdir@ +top_builddir=../.. +srcdir=@srcdir@ +@COMMENCE@ + +## Add include directory to the C preprocessor flags, add -lh5tools and +## -lhdf5 to the list of libraries. +## Add also the test include directory for #defines +CPPFLAGS=-I. -I$(srcdir) -I$(top_builddir)/src -I$(top_srcdir)/src \ + -I$(top_srcdir)/test \ + -I$(top_srcdir)/tools/lib @CPPFLAGS@ + +## Test programs and scripts. +## +TEST_PROGS=h5repacktst +TEST_SCRIPTS=$(srcdir)/h5repack.sh + + +## These are our main targets: library and tools. +## +LIBHDF5=$(top_builddir)/src/libhdf5.la +LIBTOOLS=../lib/libh5tools.la + +PUB_PROGS=h5repack +PROGS=$(PUB_PROGS) $(TEST_PROGS) + +## Source and object files for the library; do not install +## +LIB_SRC= +LIB_OBJ=$(LIB_SRC:.c=.lo) +PUB_LIB= + +## Temporary files. *.h5 are generated by h5repack. They should +## copied to the testfiles/ directory if update is required. +MOSTLYCLEAN=*.h5 + +## Source and object files for programs... +## +PROG_SRC=h5repack.c h5repack_copy.c h5repack_filters.c h5repack_refs.c h5repack_layout.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c h5repack_verify.c testh5repack_attr.c testh5repack_dset.c testh5repack_filters.c testh5repack_main.c testh5repack_make.c testh5repack_layout.c testh5repack_util.c +PROG_OBJ=$(PROG_SRC:.c=.lo) +OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_layout.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo +TEST_OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_layout.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo testh5repack_attr.lo testh5repack_dset.lo testh5repack_filters.lo testh5repack_main.lo testh5repack_make.lo testh5repack_layout.lo testh5repack_util.lo + +PRIVATE_HDR= + +## Source and object files for the tests +## +TEST_SRC= +TEST_OBJ=$(TEST_SRC:.c=.lo) + +## Programs have to be built before they can be tested! +## +check test _test: $(PROGS) + +## How to build the programs...They all depend on the hdf5 library and +## the tools library compiled in this directory. +## +$(PROGS): $(LIBTOOLS) $(LIBHDF5) + +h5repack: $(OBJS) + @$(LT_LINK_EXE) $(CFLAGS) -o $@ $(OBJS) $(LIBTOOLS) $(LIBHDF5) $(LDFLAGS) $(LIBS) + +h5repacktst: $(TEST_OBJS) + @$(LT_LINK_EXE) $(CFLAGS) -o $@ $(TEST_OBJS) $(LIBTOOLS) $(LIBHDF5) $(LDFLAGS) $(LIBS) + + +@CONCLUDE@ diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c new file mode 100644 index 0000000..bd89ca0 --- /dev/null +++ b/tools/h5repack/h5repack.c @@ -0,0 +1,497 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include "h5repack.h" + +/*------------------------------------------------------------------------- + * File: h5repack.c + * Purpose: Public API functions + *------------------------------------------------------------------------- + */ + +static int check_options(pack_opt_t *options); + + +/*------------------------------------------------------------------------- + * Function: aux_initglb_filter + * + * Purpose: auxiliary function, initialize the options global filter + * + * Return: void + * + *------------------------------------------------------------------------- + */ +static void aux_initglb_filter(pack_opt_t *options) +{ + int k; + options->filter_g.filtn = -1; + for ( k=0; k<CDVALUES; k++) + options->filter_g.cd_values[k] = -1; +} + + +/*------------------------------------------------------------------------- + * Function: h5repack + * + * Purpose: locate all high-level HDF5 objects in the file + * and compress/chunk them using options + * + * Algorythm: 2 traversals are made to the file; the 1st builds a list of + * the objects, the 2nd makes a copy of them, using the options; + * the reason for the 1st traversal is to check for invalid + * object name requests + * + * Return: 0, ok, -1, fail + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +int h5repack(const char* infile, + const char* outfile, + pack_opt_t *options) +{ + /* check input */ + if (check_options(options)<0) + return -1; + + /* check for objects in input that are in the file */ + if (check_objects(infile,options)<0) + return -1; + + /* copy the objects */ + if (copy_objects(infile,outfile,options)<0) + return -1; + + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: h5repack_init + * + * Purpose: initialize options + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_init (pack_opt_t *options, + int verbose) +{ + memset(options,0,sizeof(pack_opt_t)); + options->threshold = 1024; + options->verbose = verbose; + return (options_table_init(&(options->op_tbl))); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_end + * + * Purpose: free options table + * + *------------------------------------------------------------------------- + */ + +int h5repack_end (pack_opt_t *options) +{ + return options_table_free(options->op_tbl); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_addfilter + * + * Purpose: add a compression -f option to table + * Example: -f dset:GZIP=6 + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_addfilter(const char* str, + pack_opt_t *options) +{ + obj_list_t *obj_list=NULL; /*one object list for the -f and -c option entry */ + filter_info_t filt; /*filter info for the current -f option entry */ + int n_objs; /*number of objects in the current -f or -c option entry */ + + if (options->all_filter==1){ + printf("Error: Invalid compression input: all option is present \ + with other objects <%s>\n",str); + return -1; + } + + /* parse the -f option */ + obj_list=parse_filter(str,&n_objs,&filt,options); + if (obj_list==NULL) + { + return -1; + } + + if (options->all_filter==1) + { + /* if we are compressing all set the global filter type */ + aux_initglb_filter(options); + options->filter_g=filt; + } + + if (options->all_filter==0) + options_add_filter(obj_list,n_objs,filt,options->op_tbl); + + free(obj_list); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: h5repack_addlayout + * + * Purpose: add a layout option + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + + +int h5repack_addlayout(const char* str, + pack_opt_t *options) +{ + + obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ + int n_objs; /*number of objects in the current -t or -c option entry */ + pack_info_t pack; /*info about layout to extract from parse */ + int j; + + init_packobject(&pack); + + if (options->all_layout==1){ + printf("Error: Invalid layout input: all option \ + is present with other objects <%s>\n",str); + return -1; + } + + /* parse the layout option */ + obj_list=parse_layout(str,&n_objs,&pack,options); + if (obj_list==NULL) + return -1; + + /* set global layout option */ + if (options->all_layout==1 ) + { + options->layout_g=pack.layout; + if (pack.layout==H5D_CHUNKED) { + /* if we are chunking all set the global chunking type */ + options->chunk_g.rank=pack.chunk.rank; + for (j = 0; j < pack.chunk.rank; j++) + options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; + } + } + + if (options->all_layout==0) + options_add_layout(obj_list, + n_objs, + &pack, + options->op_tbl); + + free(obj_list); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: check_options + * + * Purpose: print options, checks for invalid options + * + * Return: void, return -1 on error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +static int check_options(pack_opt_t *options) +{ + int i, k, j, has_cp=0, has_ck=0; + + unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; + +/*------------------------------------------------------------------------- + * objects to layout + *------------------------------------------------------------------------- + */ + if (options->verbose) + { + printf("\n"); + printf("Objects to modify are...\n"); + if (options->all_layout==1) { + printf("\tApply layout to all\n "); + if (H5D_CHUNKED==options->layout_g) { + printf("with dimension ["); + for ( j = 0; j < options->chunk_g.rank; j++) + printf("%d ",(int)options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } + } + }/* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + + if (options->op_tbl->objs[i].chunk.rank>0) + { + if (options->verbose){ + printf("\t<%s> with chunk size ",name); + for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) + printf("%d ",(int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); + printf("\n"); + } + has_ck=1; + } + else if (options->op_tbl->objs[i].chunk.rank==-2) + { + if (options->verbose) + printf("\t%s %s\n",name,"NONE"); + has_ck=1; + } + } + + if (options->all_layout==1 && has_ck){ + printf("Error: Invalid chunking input: all option\ + is present with other objects\n"); + return -1; + } + +/*------------------------------------------------------------------------- + * objects to compress/uncompress + *------------------------------------------------------------------------- + */ + + if (options->verbose) + { + printf("Objects to filter are...\n"); + if (options->all_filter==1) + { + H5Z_filter_t filtn=options->filter_g.filtn; + switch (filtn) + { + case H5Z_FILTER_NONE: + printf("\tUncompress all %s\n",get_sfilter(filtn)); + break; + case H5Z_FILTER_SZIP: + printf("\tCompress all with %s compression\n",get_sfilter(filtn)); + break; + case H5Z_FILTER_DEFLATE: + printf("\tCompress all with %s compression, parameter %d\n", + get_sfilter(filtn), + options->filter_g.cd_values[0]); + break; + }; + } + } /* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + pack_info_t pack = options->op_tbl->objs[i]; + char* name = pack.path; + + for ( j=0; j<pack.nfilters; j++) + { + if (options->verbose) + { + printf("\t<%s> with %s filter", + name, + get_sfilter(pack.filter[j].filtn)); + } + + has_cp=1; + + /*check for invalid combination of options */ + switch (pack.filter[j].filtn) + { + default: + break; + case H5Z_FILTER_SZIP: + + szip_pixels_per_block=pack.filter[j].cd_values[0]; + + /* check szip parameters */ + if ( pack.chunk.rank!=-1 /* + it means a layout was not input, so there is no + case to try to check it + */ + && check_szip(0, /* do not test size */ + pack.chunk.rank, + pack.chunk.chunk_lengths, + szip_options_mask, + &szip_pixels_per_block, + options)==0) + { + /* Return: 1=can apply the filter + 0=cannot apply the filter + Reset this object filter info + */ + + options->op_tbl->objs[i].filter[j].filtn=-1; + options->op_tbl->objs[i].chunk.rank=-1; + printf("\tObject <%s> cannot be filtered\n",name); + + + } + + break; + } /* switch */ + } /* j */ + } /* i */ + + if (options->all_filter==1 && has_cp){ + printf("Error: Invalid compression input: all option\ + is present with other objects\n"); + return -1; + } + return 0; +} + +/*------------------------------------------------------------------------- + * Function: read_info + * + * Purpose: read comp and chunk options from file + * + * Return: void, exit on error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ + +void read_info(const char *filename, + pack_opt_t *options) +{ + + char stype[10]; + char comp_info[1024]; + FILE *fp; + char c; + int i, rc=1; + char *srcdir = getenv("srcdir"); /* the source directory */ + char data_file[512]=""; /* buffer to hold name of existing file */ + + /* compose the name of the file to open, using the srcdir, if appropriate */ + if (srcdir){ + strcpy(data_file,srcdir); + strcat(data_file,"/"); + } + strcat(data_file,filename); + + + if ((fp = fopen(data_file, "r")) == (FILE *)NULL) { + printf( "Cannot open options file %s", filename); + exit(1); + } + + /* cycle until end of file reached */ + while( 1 ) + { + rc=fscanf(fp, "%s", stype); + if (rc==-1) + break; + + /*------------------------------------------------------------------------- + * filter + *------------------------------------------------------------------------- + */ + if (strcmp(stype,"-f") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + if (feof(fp)) break; + } + c='0'; + /* go until end */ + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + if (feof(fp)) break; + if (c==10 /*eol*/) break; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addfilter(comp_info,options)==-1){ + printf( "Could not add compression option. Exiting\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + else if (strcmp(stype,"-l") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + if (feof(fp)) break; + } + c='0'; + /* go until end */ + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + if (feof(fp)) break; + if (c==10 /*eol*/) break; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addlayout(comp_info,options)==-1){ + printf( "Could not add chunck option. Exiting\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * not valid + *------------------------------------------------------------------------- + */ + else { + printf( "Bad file format for %s", filename); + exit(1); + } + } + + fclose(fp); + return; +} + + diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h new file mode 100644 index 0000000..b91d516 --- /dev/null +++ b/tools/h5repack/h5repack.h @@ -0,0 +1,362 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#ifndef H5REPACK_H__ +#define H5REPACK_H__ + +#include "hdf5.h" +#include "h5trav.h" +#include "h5diff.h" +#include "h5tools.h" + +#define H5FOPENERROR "unable to open file" + +#define PFORMAT "%-7s %-7s %-7s\n" /*chunk info, compression info, name*/ +#define PFORMAT1 "%-7s %-7s %-7s" /*chunk info, compression info, name*/ + +#define MAX_NC_NAME 256 /* max length of a name */ +#define MAX_VAR_DIMS 32 /* max per variable dimensions */ + + +/*------------------------------------------------------------------------- + * data structures for command line options + *------------------------------------------------------------------------- + */ + +/* a list of names */ +typedef struct { + char obj[MAX_NC_NAME]; +} obj_list_t; + +/* + the type of filter and additional parameter + type can be one of the filters + H5Z_FILTER_NONE 0, uncompress if compressed + H5Z_FILTER_DEFLATE 1 , deflation like gzip + H5Z_FILTER_SHUFFLE 2 , shuffle the data + H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC + H5Z_FILTER_SZIP 4 , szip compression +*/ + +#define CDVALUES 2 + +typedef struct { + H5Z_filter_t filtn; /* filter identification number */ + int cd_values[CDVALUES]; /* filter client data values */ +} filter_info_t; + +/* chunk lengths along each dimension and rank */ +typedef struct { + hsize_t chunk_lengths[MAX_VAR_DIMS]; + int rank; +} chunk_info_t; + +/* we currently define a maximum value for the filters array, + that corresponds to the current library filters */ +#define H5_REPACK_MAX_NFILTERS 4 + +/* information for one object, contains PATH, CHUNK info and FILTER info */ +typedef struct { + char path[MAX_NC_NAME]; /* name of object */ + filter_info_t filter[H5_REPACK_MAX_NFILTERS]; /* filter array */ + int nfilters; /* current number of filters */ + H5D_layout_t layout; /* layout information */ + chunk_info_t chunk; /* chunk information */ + hid_t refobj_id; /* object ID, references */ +} pack_info_t; + +/* store a table of all objects */ +typedef struct { + int size; + int nelems; + pack_info_t *objs; +} pack_opttbl_t; + + +/*------------------------------------------------------------------------- + * command line options + *------------------------------------------------------------------------- + */ + +/* all the above, ready to go to the hrepack call */ +typedef struct { + pack_opttbl_t *op_tbl; /*table with all -c and -f options */ + int all_layout; /*apply the layout to all objects */ + int all_filter; /*apply the filter to all objects */ + filter_info_t filter_g; /*global filter INFO for the ALL case */ + chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /*global layout information for the ALL case */ + int verbose; /*verbose mode */ + hsize_t threshold; /*minimum size to compress, in bytes */ +} pack_opt_t; + + + +/*------------------------------------------------------------------------- + * public functions + *------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +extern "C" { +#endif + +int h5repack (const char* infile, const char* outfile, pack_opt_t *options); +int h5repack_addfilter (const char* str, pack_opt_t *options); +int h5repack_addlayout (const char* str, pack_opt_t *options); +int h5repack_init (pack_opt_t *options, int verbose); +int h5repack_end (pack_opt_t *options); +int h5repack_verify (const char *fname,pack_opt_t *options); +int h5repack_cmpdcpl (const char *fname1, + const char *fname2); + + +#ifdef __cplusplus +} +#endif + + + +/*------------------------------------------------------------------------- + * private functions + *------------------------------------------------------------------------- + */ + + +int check_objects(const char* fname, + pack_opt_t *options); + +int copy_objects(const char* fnamein, + const char* fnameout, + pack_opt_t *options); + +void print_objlist(const char *filename, + int nobjects, + trav_info_t *travi ); + +int do_copy_objects(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options); + +int copy_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options + ); + +int do_copy_refobjs(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options); /* repack options */ + + + +void read_info(const char *filename,pack_opt_t *options); +void init_packobject(pack_info_t *obj); +int print_filters(hid_t dcpl_id); + + + +/*------------------------------------------------------------------------- + * filters + *------------------------------------------------------------------------- + */ + +int filter_this(const char* name, + pack_opt_t *options, + pack_info_t *pack); /* info about object to filter */ + +int apply_filters(const char* name, /* object name from traverse list */ + int rank, /* rank of dataset */ + hsize_t *dims, /* dimensions of dataset */ + hid_t dcpl_id, /* dataset creation property list */ + hid_t type_id, /* datatype */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj); /* info about object to filter */ + +int has_filter(hid_t dcpl_id, + H5Z_filter_t filtnin); + +int check_szip_params( unsigned bits_per_pixel, + unsigned pixels_per_block, + unsigned pixels_per_scanline, + hsize_t image_pixels); + +int check_szip(hid_t type_id, /* dataset datatype */ + int rank, /* chunk rank */ + hsize_t *dims, /* chunk dims */ + unsigned szip_options_mask /*IN*/, + unsigned *szip_pixels_per_block /*IN,OUT*/, + pack_opt_t *options); + +int can_read(const char* name, /* object name from traverse list */ + hid_t dcpl_id, /* dataset creation property list */ + pack_opt_t *options); /* repack options */ + + +/*------------------------------------------------------------------------- + * layout functions + *------------------------------------------------------------------------- + */ + +int has_layout(hid_t dcpl_id, + pack_info_t *obj); + +int layout_this(hid_t dcpl_id, /* DCPL from input object */ + const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *pack /*OUT*/) /* object to apply layout */; + +int apply_layout(hid_t dcpl_id, + pack_info_t *pack); /* info about object */ + + +/*------------------------------------------------------------------------- + * options table + *------------------------------------------------------------------------- + */ +int options_table_init( pack_opttbl_t **tbl ); +int options_table_free( pack_opttbl_t *table ); +int options_add_layout( obj_list_t *obj_list, + int n_objs, + pack_info_t *pack, + pack_opttbl_t *table ); +int options_add_filter ( obj_list_t *obj_list, + int n_objs, + filter_info_t filt, + pack_opttbl_t *table ); +pack_info_t* options_get_object( const char *path, + pack_opttbl_t *table); + +/*------------------------------------------------------------------------- + * parse functions + *------------------------------------------------------------------------- + */ + +obj_list_t* parse_filter(const char *str, + int *n_objs, + filter_info_t *filt, + pack_opt_t *options); + +obj_list_t* parse_layout(const char *str, + int *n_objs, + pack_info_t *pack, /* info about object */ + pack_opt_t *options); + +const char* get_sfilter (H5Z_filter_t filtn); +int parse_number(char *str); + +/*------------------------------------------------------------------------- + * tests + *------------------------------------------------------------------------- + */ + +#define FNAME1 "test1.h5" +#define FNAME1OUT "test1out.h5" +#define FNAME2 "test2.h5" +#define FNAME2OUT "test2out.h5" +#define FNAME3 "test3.h5" +#define FNAME3OUT "test3out.h5" +#define FNAME4 "test4.h5" +#define FNAME4OUT "test4out.h5" +#define FNAME5 "test5.h5" +#define FNAME5OUT "test5out.h5" +#define FNAME6 "test6.h5" + +int make_testfiles(void); + +int write_dset( hid_t loc_id, + int rank, + hsize_t *dims, + const char *dset_name, + hid_t type_id, + void *buf ); +int write_attr(hid_t loc_id, + int rank, + hsize_t *dims, + const char *attr_name, + hid_t type_id, + void *buf); +void write_attr_in(hid_t loc_id, + const char* dset_name, /* for saving reference to dataset*/ + hid_t fid, /* for reference create */ + int make_diffs /* flag to modify data buffers */); +void write_dset_in(hid_t loc_id, + const char* dset_name, /* for saving reference to dataset*/ + hid_t file_id, + int make_diffs /* flag to modify data buffers */); + +int make_filters(hid_t loc_id); +int make_layout(hid_t loc_id); + + +/*------------------------------------------------------------------------- + * tests utils + *------------------------------------------------------------------------- + */ +int make_dset(hid_t loc_id, + const char *name, + hid_t sid, + hid_t dcpl, + void *buf); + +int make_attr(hid_t loc_id, + int rank, + hsize_t *dims, + const char *attr_name, + hid_t type_id, + void *buf); + + + +/*------------------------------------------------------------------------- + * check SZIP parameters + *------------------------------------------------------------------------- + */ + +typedef struct +{ + int compression_mode; + int options_mask; + unsigned bits_per_pixel; + unsigned pixels_per_block; + unsigned pixels_per_scanline; + hsize_t pixels; + +}szip_comp_t; + +/* for SZIP */ +#if !defined (NN_OPTION_MASK) +#define NN_OPTION_MASK 32 +#endif +#if !defined (RAW_OPTION_MASK) +#define RAW_OPTION_MASK 128 +#endif +#if !defined (MAX_BLOCKS_PER_SCANLINE) +#define MAX_BLOCKS_PER_SCANLINE 128 +#endif +#if !defined (MAX_PIXELS_PER_BLOCK) +#define MAX_PIXELS_PER_BLOCK 32 +#endif +#if !defined (MAX_PIXELS_PER_SCANLINE) +#define MAX_PIXELS_PER_SCANLINE (MAX_BLOCKS_PER_SCANLINE)*(MAX_PIXELS_PER_BLOCK) +#endif +#if !defined (NN_MODE) +#define NN_MODE 1 +#endif + + + +#endif /* H5REPACK_H__ */ diff --git a/tools/h5repack/h5repack.sh b/tools/h5repack/h5repack.sh new file mode 100644 index 0000000..f1a6281 --- /dev/null +++ b/tools/h5repack/h5repack.sh @@ -0,0 +1,178 @@ +#! /bin/sh +# +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. +# +# Tests for the h5repack tool + +H5REPACK=h5repack # The tool name +H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary + +H5DIFF=../h5diff/h5diff # The h5diff tool name +H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary + +nerrors=0 +verbose=yes + +# The build (current) directory might be different than the source directory. +# +if test -z "$srcdir"; then + srcdir=. +fi + +test -d ../testfiles || mkdir ../testfiles + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + SPACES=" " + echo "Testing h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Call the h5diff tool +# +DIFFTEST() +{ + VERIFY $@ + if [ "`uname -s`" = "TFLOPS O/S" ]; then + $RUNSERIAL $H5DIFF_BIN $@ + else + $RUNSERIAL $H5DIFF_BIN "$@" + fi + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + +} + +# Call h5repack +# +TOOLTEST() +{ + # Run test. + # Tflops interprets "$@" as "" when no parameter is given (e.g., the + # case of missing file name). Changed it to use $@ till Tflops fixes it. + TESTING $H5REPACK $@ + + infile=$srcdir/../testfiles/$1 + outfile=out.$1 + shift + if [ "`uname -s`" = "TFLOPS O/S" ]; then + $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@ + else + $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" + fi + + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + rm -f $outfile +} + + +# +# The tests +# We use the files generated by h5repacktst +# Each run generates "file4.out.h5" and the tool h5diff is used to +# compare the input and output files +# + +# copy files +TOOLTEST test1.h5 +TOOLTEST test3.h5 +TOOLTEST test4.h5 + +#TOOLTEST test5.h5 + +# remove all filters +TOOLTEST test4.h5 -f NONE + +# remove one filter +TOOLTEST test4.h5 -f dset_gzip:NONE + +# gzip +TOOLTEST test4.h5 -f dset1:GZIP=9 +TOOLTEST test4.h5 -f GZIP=1 + +# szip +TOOLTEST test4.h5 -f dset1:SZIP=8 +TOOLTEST test4.h5 -f SZIP=8 + +# shuffle +TOOLTEST test4.h5 -f dset1:SHUF +TOOLTEST test4.h5 -f SHUF + +# fletcher +TOOLTEST test4.h5 -f dset1:FLET +TOOLTEST test4.h5 -f FLET + +#layout chunk +TOOLTEST test4.h5 -l dset1:CHUNK=20x10 +TOOLTEST test4.h5 -l CHUNK=20x10 + +#layout compact +TOOLTEST test4.h5 -l dset1:COMPA +TOOLTEST test4.h5 -l COMPA + +#layout contiguous +TOOLTEST test4.h5 -l dset1:CONTI +TOOLTEST test4.h5 -l CONTI + +#conversions +TOOLTEST test4.h5 -l dset_compact:CONTI +TOOLTEST test4.h5 -l dset_compact:CHUNK=2x5 +TOOLTEST test4.h5 -l dset_compact:COMPA +TOOLTEST test4.h5 -l dset_contiguous:COMPA +TOOLTEST test4.h5 -l dset_contiguous:CHUNK=3x6 +TOOLTEST test4.h5 -l dset_contiguous:CONTI +TOOLTEST test4.h5 -l dset_chunk:COMPA +TOOLTEST test4.h5 -l dset_chunk:CONTI +TOOLTEST test4.h5 -l dset_chunk:CHUNK=18x13 + +#filters +TOOLTEST test4.h5 -f dset1:SHUF -f dset1,dset2:GZIP=6 +TOOLTEST test4.h5 -l dset1:CHUNK=20x10 -f dset1,dset2:SZIP=8 + +#filter conversions +TOOLTEST test4.h5 -f dset_gzip:SZIP=8 +TOOLTEST test4.h5 -f dset_szip:GZIP=1 +TOOLTEST test4.h5 -f dset_all:GZIP=1 + +#limit +TOOLTEST test4.h5 -f GZIP=1 -m 1024 + +#file +TOOLTEST test4.h5 -e ../testfiles/h5repack_info.txt + +if test $nerrors -eq 0 ; then + echo "All $H5REPACK tests passed." +fi + +exit $nerrors diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c new file mode 100644 index 0000000..7763bd2 --- /dev/null +++ b/tools/h5repack/h5repack_copy.c @@ -0,0 +1,620 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include "H5private.h" +#include "h5repack.h" + +/*------------------------------------------------------------------------- + * Function: copy_objects + * + * Purpose: duplicate all HDF5 objects in the file + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 23, 2003 + * + *------------------------------------------------------------------------- + */ + +int copy_objects(const char* fnamein, + const char* fnameout, + pack_opt_t *options) +{ + hid_t fidin; + hid_t fidout; + trav_table_t *travt=NULL; + +/*------------------------------------------------------------------------- + * open the files + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY { + + /* Open the files */ + if ((fidin=H5Fopen(fnamein,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ){ + printf("h5repack: <%s>: %s\n", fnamein, H5FOPENERROR ); + exit(1); + } + if ((fidout=H5Fcreate(fnameout,H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))<0 ){ + printf("h5repack: <%s>: Could not create file\n", fnameout ); + exit(1); + } + /* enable error reporting */ + } H5E_END_TRY; + + + if (options->verbose) + printf("Making file <%s>...\n",fnameout); + + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fidin,travt)<0) + goto out; + +/*------------------------------------------------------------------------- + * do the copy + *------------------------------------------------------------------------- + */ + if(do_copy_objects(fidin,fidout,travt,options)<0) { + printf("h5repack: <%s>: Could not copy data to: %s\n", fnamein, fnameout); + goto out; + } + +/*------------------------------------------------------------------------- + * do the copy of referenced objects + * and create hard links + *------------------------------------------------------------------------- + */ + if(do_copy_refobjs(fidin,fidout,travt,options)<0) { + printf("h5repack: <%s>: Could not copy data to: %s\n", fnamein, fnameout); + goto out; + } + + /* free table */ + trav_table_free(travt); + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + H5Fclose(fidin); + H5Fclose(fidout); + return 0; + +/*------------------------------------------------------------------------- + * out + *------------------------------------------------------------------------- + */ + +out: + H5E_BEGIN_TRY { + H5Fclose(fidin); + H5Fclose(fidout); + trav_table_free(travt); + } H5E_END_TRY; + + return -1; +} + + + + +/*------------------------------------------------------------------------- + * Function: do_copy_objects + * + * Purpose: duplicate all HDF5 objects in the file + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 23, 2003 + * + *------------------------------------------------------------------------- + */ + +int do_copy_objects(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options) /* repack options */ +{ + hid_t grp_in=-1; /* group ID */ + hid_t grp_out=-1; /* group ID */ + hid_t dset_in=-1; /* read dataset ID */ + hid_t dset_out=-1; /* write dataset ID */ + hid_t type_in=-1; /* named type ID */ + hid_t type_out=-1; /* named type ID */ + hid_t dcpl_id=-1; /* dataset creation property list ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file data type ID */ + hid_t mtype_id=-1; /* memory data type ID */ + size_t msize; /* memory size of memory type */ + void *buf=NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ +#ifdef LATER + hsize_t dsize_out; /* output dataset size after filter */ +#endif /* LATER */ + int i, j; + +/*------------------------------------------------------------------------- + * copy the suppplied object list + *------------------------------------------------------------------------- + */ + + for ( i = 0; i < travt->nobjs; i++) + { + + buf=NULL; + switch ( travt->objs[i].type ) + { +/*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + case H5G_GROUP: + if (options->verbose) + printf(" %-10s %s\n", "group",travt->objs[i].name ); + + if ((grp_out=H5Gcreate(fidout,travt->objs[i].name, 0))<0) + goto error; + + if((grp_in = H5Gopen (fidin,travt->objs[i].name))<0) + goto error; + + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(grp_in,grp_out,options)<0) + goto error; + + if (H5Gclose(grp_out)<0) + goto error; + if (H5Gclose(grp_in)<0) + goto error; + + + break; + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + case H5G_DATASET: + if (options->verbose) + printf(" %-10s %s\n", "dataset",travt->objs[i].name ); + + if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) + goto error; + if ((space_id=H5Dget_space(dset_in))<0) + goto error; + if ((ftype_id=H5Dget_type (dset_in))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) + goto error; + HDmemset(dims, 0, sizeof dims); + if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) + goto error; + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0) + goto error; + if ((msize=H5Tget_size(mtype_id))==0) + goto error; + + if (options->verbose) + print_filters(dcpl_id); + +/*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- + */ + if (h5tools_canreadf((options->verbose?travt->objs[i].name:NULL),dcpl_id)==1) + { + +/*------------------------------------------------------------------------- + * references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * in a second traversal of the output file + *------------------------------------------------------------------------- + */ + if ( (H5T_REFERENCE!=H5Tget_class(mtype_id))) + { + /* the information about the object to be filtered/"layouted" */ + pack_info_t obj; + init_packobject(&obj); + + /* get the storage size of the input dataset */ + dsize_in=H5Dget_storage_size(dset_in); + + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + + /*------------------------------------------------------------------------- + * apply the layout; check first if the object is to be modified. + *------------------------------------------------------------------------- + */ + if (layout_this(dcpl_id,travt->objs[i].name,options,&obj)) + { + obj.chunk.rank=rank; + if (apply_layout(dcpl_id,&obj)<0) + goto error; + } + + /*------------------------------------------------------------------------- + * apply the filter; check if the object is to be filtered. + *------------------------------------------------------------------------- + */ + if (apply_filters(travt->objs[i].name,rank,dims,dcpl_id,mtype_id,options,&obj)<0) + goto error; + + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + goto error; + if (dsize_in && nelmts) { + if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + } + /*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(dset_in,dset_out,options)<0) + goto error; + +#ifdef LATER + /*------------------------------------------------------------------------- + * store the storage sizes + *------------------------------------------------------------------------- + */ + + dsize_out=H5Dget_storage_size(dset_out); +#endif /* LATER */ + + /*close */ + if (H5Dclose(dset_out)<0) + goto error; + + if (buf) + free(buf); + + }/*H5T_STD_REF_OBJ*/ + }/*can_read*/ + + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Tclose(ftype_id)<0) + goto error; + if (H5Tclose(mtype_id)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_in)<0) + goto error; + + break; + +/*------------------------------------------------------------------------- + * H5G_TYPE + *------------------------------------------------------------------------- + */ + case H5G_TYPE: + + if ((type_in = H5Topen (fidin,travt->objs[i].name))<0) + goto error; + + if ((type_out = H5Tcopy(type_in))<0) + goto error; + + if ((H5Tcommit(fidout,travt->objs[i].name,type_out))<0) + goto error; + +/*------------------------------------------------------------------------- + * copy attrs + *------------------------------------------------------------------------- + */ + if (copy_attr(type_in,type_out,options)<0) + goto error; + + if (H5Tclose(type_in)<0) + goto error; + if (H5Tclose(type_out)<0) + goto error; + + if (options->verbose) + printf(" %-10s %s\n","datatype",travt->objs[i].name ); + + break; + + +/*------------------------------------------------------------------------- + * H5G_LINK + *------------------------------------------------------------------------- + */ + + case H5G_LINK: + { + H5G_stat_t statbuf; + char *targbuf=NULL; + + if (H5Gget_objinfo(fidin,travt->objs[i].name,FALSE,&statbuf)<0) + goto error; + + targbuf = malloc(statbuf.linklen); + + if (H5Gget_linkval(fidin,travt->objs[i].name,statbuf.linklen,targbuf)<0) + goto error; + + if (H5Glink(fidout, + H5G_LINK_SOFT, + targbuf, /* current name of object */ + travt->objs[i].name /* new name of object */ + )<0) + goto error; + + free(targbuf); + + if (options->verbose) + printf(" %-10s %s\n","link",travt->objs[i].name ); + + } + break; + + default: + if (options->verbose) + printf(" %-10s %s\n","User defined object",travt->objs[i].name); + break; + } + } + +/*------------------------------------------------------------------------- + * the root is a special case, we get an ID for the root group + * and copy its attributes using that ID + * it must be done last, because the attributes might contain references to + * objects in the object list + *------------------------------------------------------------------------- + */ + + if ((grp_out = H5Gopen(fidout,"/"))<0) + goto error; + + if ((grp_in = H5Gopen(fidin,"/"))<0) + goto error; + + if (copy_attr(grp_in,grp_out,options)<0) + goto error; + + if (H5Gclose(grp_out)<0) + goto error; + if (H5Gclose(grp_in)<0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Gclose(grp_in); + H5Gclose(grp_out); + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_in); + H5Dclose(dset_out); + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Tclose(type_in); + H5Tclose(type_out); + if (buf) + free(buf); + } H5E_END_TRY; + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: copy_attr + * + * Purpose: copy attributes located in LOC_IN, which is obtained either from + * loc_id = H5Gopen( fid, name); + * loc_id = H5Dopen( fid, name); + * loc_id = H5Topen( fid, name); + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 28, 2003 + * + *------------------------------------------------------------------------- + */ + +int copy_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options + ) +{ + hid_t attr_id=-1; /* attr ID */ + hid_t attr_out=-1; /* attr ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file data type ID */ + hid_t mtype_id=-1; /* memory data type ID */ + size_t msize; /* memory size of type */ + void *buf=NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + int n, j; + unsigned u; + + if ((n = H5Aget_num_attrs(loc_in))<0) + goto error; + + for ( u = 0; u < (unsigned)n; u++) + { + + /* set data buffer to NULL each iteration + we might not use it in the case of references + */ + buf=NULL; +/*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + /* open attribute */ + if ((attr_id = H5Aopen_idx(loc_in, u))<0) + goto error; + + /* get name */ + if (H5Aget_name( attr_id, 255, name )<0) + goto error; + + /* get the file datatype */ + if ((ftype_id = H5Aget_type( attr_id )) < 0 ) + goto error; + + /* get the dataspace handle */ + if ((space_id = H5Aget_space( attr_id )) < 0 ) + goto error; + + /* get dimensions */ + if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) + goto error; + + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0) + goto error; + if ((msize=H5Tget_size(mtype_id))==0) + goto error; + +/*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * this is done on a second sweep of the file that just copies + * the referenced objects + *------------------------------------------------------------------------- + */ + if ( ! H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + { + + + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Aread(attr_id,mtype_id,buf)<0) + goto error; + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) + goto error; + if(H5Awrite(attr_out,mtype_id,buf)<0) + goto error; + + /*close*/ + if (H5Aclose(attr_out)<0) + goto error; + + + if (buf) + free(buf); + + + } /*H5T_STD_REF_OBJ*/ + + + if (options->verbose) + printf(" %-13s %s\n", "attr", name); + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id)<0) goto error; + if (H5Tclose(mtype_id)<0) goto error; + if (H5Sclose(space_id)<0) goto error; + if (H5Aclose(attr_id)<0) goto error; + + } /* u */ + + return 0; + +error: + H5E_BEGIN_TRY { + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + if (buf) + free(buf); + } H5E_END_TRY; + return -1; +} + diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c new file mode 100644 index 0000000..c8b9b5d --- /dev/null +++ b/tools/h5repack/h5repack_filters.c @@ -0,0 +1,541 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + +/*------------------------------------------------------------------------- + * Function: aux_objinsert_filter + * + * Purpose: auxiliary function, inserts the filter in object OBJ + * + * Return: void + * + *------------------------------------------------------------------------- + */ +static void aux_objinsert_filter(pack_info_t *obj, + filter_info_t filt) +{ + obj->nfilters=1; + obj->filter[0]=filt; + +} + +/*------------------------------------------------------------------------- + * Function: filter_this + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the filter information OBJ + * + * Return: 0 not found, 1 found + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ +int filter_this(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ +{ + char *pdest; + int result; + int i, j; + + /* if we are applying to all objects just return true */ + if (options->all_filter) + { + /* assign the global filter and chunk info to the OBJ info */ + aux_objinsert_filter( obj, options->filter_g ); + obj->chunk=options->chunk_g; + return 1; + } + + for ( i=0; i<options->op_tbl->nelems; i++) + { + for ( j=0; j<options->op_tbl->objs[i].nfilters; j++) + { + if (options->op_tbl->objs[i].filter[j].filtn != -1 ) + { + if (strcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + + pdest = strstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + } /*if*/ + }/*j*/ + }/*i*/ + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: apply_filters + * + * Purpose: apply the filters in the object to the property list; + * do extra checking in the case of SZIP; delete all filters in the case + * of H5Z_FILTER_NONE present in the PACK_INFO_T filter array + * + * Return: 0 success, -1 an error occured + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ +int apply_filters(const char* name, /* object name from traverse list */ + int rank, /* rank of dataset */ + hsize_t *dims, /* dimensions of dataset */ + hid_t dcpl_id, /* dataset creation property list */ + hid_t type_id, /* dataset datatype */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj) /* info about object to filter */ +{ + int nfilters; /* number of filters in DCPL */ + unsigned aggression; /* the deflate level */ + hsize_t nelmts; /* number of elements in dataset */ + size_t size; /* size of datatype in bytes */ + unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; + int i; + + /* check first if the object is to be filtered */ + if (filter_this(name,options,obj)==0) + return 0; + + if (rank==0) + goto out; + + /* check for datasets too small */ + if ((size=H5Tget_size(type_id))==0) + return 0; + nelmts=1; + for (i=0; i<rank; i++) + nelmts*=dims[i]; + if (nelmts*size < options->threshold ) + { + if (options->verbose) + printf("Warning: Filter not applied to <%s>. Dataset smaller than <%d> bytes\n", + name,(int)options->threshold); + return 0; + } + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; +/*------------------------------------------------------------------------- + * check if we have filters in the pipeline + * we want to replace them with the input filters + *------------------------------------------------------------------------- + */ + if (nfilters) { + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } +/*------------------------------------------------------------------------- + * filters require CHUNK layout; if we do not have one define a default + *------------------------------------------------------------------------- + */ + if (obj->chunk.rank<=0) + { + obj->chunk.rank=rank; + for (i=0; i<rank; i++) + obj->chunk.chunk_lengths[i] = dims[i]; + } + +/*------------------------------------------------------------------------- + * the type of filter and additional parameter + * type can be one of the filters + * H5Z_FILTER_NONE 0, uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + *------------------------------------------------------------------------- + */ + for ( i=0; i<obj->nfilters; i++) + { + switch (obj->filter[i].filtn) + { + default: + break; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_DEFLATE: + aggression=obj->filter[i].cd_values[0]; + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + break; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP 4 , szip compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SZIP: + szip_pixels_per_block=obj->filter[i].cd_values[0]; + /* check szip parameters */ + if (check_szip(type_id, + obj->chunk.rank, + obj->chunk.chunk_lengths, + szip_options_mask, + &szip_pixels_per_block, + options)==1) + { + /* set up for szip data */ + if(H5Pset_chunk(dcpl_id,obj->chunk.rank,obj->chunk.chunk_lengths)<0) + return -1; + if (H5Pset_szip(dcpl_id, szip_options_mask, szip_pixels_per_block)<0) + return -1; + } + else + { + if (options->verbose) + printf("Warning: SZIP filter cannot be applied to <%s>\n",name); + } + break; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SHUFFLE: + if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0) + return -1; + if (H5Pset_shuffle(dcpl_id)<0) + return -1; + break; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_FLETCHER32: + if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0) + return -1; + if (H5Pset_fletcher32(dcpl_id)<0) + return -1; + break; + } /* switch */ + }/*i*/ + + return 0; + +out: + if (options->verbose) + printf("Warning: Filter could not be applied to <%s>\n",name); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: print_filters + * + * Purpose: print the filters in DCPL + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int print_filters(hid_t dcpl_id) +{ + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + size_t cd_num; /* filter client data counter */ + char f_name[256]; /* filter/file name */ + char s[64]; /* temporary string buffer */ + int i; + + /* get information about filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + for (i=0; i<nfilters; i++) + { + cd_nelmts = NELMTS(cd_values); + filtn = H5Pget_filter(dcpl_id, + (unsigned)i, + &filt_flags, + &cd_nelmts, + cd_values, + sizeof(f_name), + f_name); + + f_name[sizeof(f_name)-1] = '\0'; + sprintf(s, "Filter-%d:", i); + printf(" %-10s %s-%u %s {", s, + f_name[0]?f_name:"method", + (unsigned)filtn, + filt_flags & H5Z_FLAG_OPTIONAL?"OPT":""); + for (cd_num=0; cd_num<cd_nelmts; cd_num++) { + printf("%s%u", cd_num?", ":"", cd_values[cd_num]); + } + printf("}\n"); + } + + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: check_szip + * + * Purpose: utility to check SZIP parameters + * + * SZIP compresses data block by block, with a user-tunable block size. + * This block size is passed in the parameter pixels_per_block and must be even, + * with typical values being 8, 10, 16, and 32. The more pixel values vary, + * the smaller this number should be. For optimal performance, the number of + * pixels per scan line (i.e., the size of the fastest-changing dimension in the chunk) + * should be an even multiple of the number of pixels per block. + * + * Return: 1=can apply the filter + * 0=cannot apply the filter + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 23, 2003 + * + *------------------------------------------------------------------------- + */ + +int check_szip(hid_t type_id, /* dataset datatype */ + int rank, /* chunk rank */ + hsize_t *dims, /* chunk dims */ + unsigned szip_options_mask /*IN*/, + unsigned *szip_pixels_per_block /*IN,OUT*/, + pack_opt_t *options) +{ + size_t size=0; /* size of datatype in bytes */ + szip_comp_t szip; + int i; + unsigned ppb=*szip_pixels_per_block; + + if (type_id) + { + if ((size=H5Tget_size(type_id))==0) + return 0; + + switch (H5Tget_class(type_id)) + { + default: + return 0; + case H5T_INTEGER: + case H5T_FLOAT: + break; + } + } + + + /* + pixels_per_scanline = size of the fastest-changing dimension + Must be <= MAX_PIXELS_PER_SCANLINE and <= pixels + */ + szip.pixels_per_scanline = (unsigned)dims[rank-1]; + szip.pixels = 1; + for ( i = 0; i < rank; i++) + { + szip.pixels *= dims[i]; + } + + if (szip.pixels_per_scanline > MAX_PIXELS_PER_SCANLINE) + { + printf("Warning: in SZIP setting, pixels per scanline was set to <%d>, \ + MAX_PIXELS_PER_SCANLINE\n",MAX_PIXELS_PER_SCANLINE); + szip.pixels_per_scanline = MAX_PIXELS_PER_SCANLINE; + } + + /* + pixels_per_block must be an even number, and <= pixels_per_scanline + and <= MAX_PIXELS_PER_BLOCK + */ + + if (ppb > szip.pixels_per_scanline) + { + /* set ppb to pixels per scanline and try to make it an even number */ + ppb=szip.pixels_per_scanline; + if (ppb%2!=0) + ppb--; + if (ppb<=1 ) + { + printf("Warning: in SZIP settings, pixels per block <%d>,\ + cannot be set with pixels per scanline <%d>\n", + ppb, szip.pixels_per_scanline); + return 0; + } + else + { + if (options->verbose) + printf("Warning: In SZIP settings, pixels per block was set to <%d>\n", ppb); + } + } + szip.pixels_per_block = ppb; + *szip_pixels_per_block = ppb; + + szip.options_mask = szip_options_mask; + szip.compression_mode = NN_MODE; + + /* + bits_per_pixel + Must be in range 1..24,32,64 + */ + szip.bits_per_pixel = 0; + if (type_id) { + switch(size) + { + case 1: + szip.bits_per_pixel = 8; + break; + case 2: + szip.bits_per_pixel = 16; + break; + case 4: + szip.bits_per_pixel = 32; + break; + case 8: + szip.bits_per_pixel = 64; + break; + default: + printf("Warning: Invalid numeric type of size <%u> for SZIP\n",(unsigned)size); + return 0; + }} + + return check_szip_params( szip.bits_per_pixel, + szip.pixels_per_block, + szip.pixels_per_scanline, + szip.pixels); + +} + + +/*------------------------------------------------------------------------- + * Function: check_szip_params + * + * Purpose: Adapted from rice.c. Checks the SZIP parameters + * + * Return: 1=can apply the filter + * 0=cannot apply the filter + * + *------------------------------------------------------------------------- + */ + +int check_szip_params( unsigned bits_per_pixel, + unsigned pixels_per_block, + unsigned pixels_per_scanline, + hsize_t image_pixels) +{ + + if (pixels_per_block & 1) + { + printf("Pixels per block must be even.\n"); + return 0; + } + + if (pixels_per_block > pixels_per_scanline) + { + printf("Pixels per block is greater than pixels per scanline.\n"); + return 0; + } + + if (bits_per_pixel) /* if provided for test */ + { + if (bits_per_pixel >= 1 && bits_per_pixel <= 24) + ; + else if (bits_per_pixel == 32 || bits_per_pixel == 64) + ; + else + { + printf("bits per pixel must be in range 1..24,32,64"); + return 0; + } + } + + if (pixels_per_block > MAX_PIXELS_PER_BLOCK) + { + printf("maximum pixels per block exceeded"); + return 0; + } + + if (pixels_per_block & 1) + { + printf("pixels per block must be even"); + return 0; + } + + if (pixels_per_block > pixels_per_scanline) + { + printf("pixels per block > pixels per scanline"); + return 0; + } + + if (pixels_per_scanline > MAX_PIXELS_PER_SCANLINE) + { + printf("maximum pixels per scanline exceeded"); + return 0; + } + + if (image_pixels < pixels_per_scanline) + { + printf("image pixels less than pixels per scanline"); + return 0; + } + + if (image_pixels % pixels_per_scanline) + { + printf("Pixels (%d) must be integer multiple of pixels per scanline (%d)\n", + (unsigned)image_pixels,pixels_per_scanline); + return 0; + } + +#if 0 + if (pixels_per_scanline % pixels_per_block) + { + printf("Pixels per scanline (%d) must be an integer multiple of pixels per block (%d)\n", + pixels_per_scanline, pixels_per_block); + return 0; + } +#endif + + return 1; +} + diff --git a/tools/h5repack/h5repack_list.c b/tools/h5repack/h5repack_list.c new file mode 100644 index 0000000..74e79a4 --- /dev/null +++ b/tools/h5repack/h5repack_list.c @@ -0,0 +1,159 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include "H5private.h" +#include "h5repack.h" + + +/*------------------------------------------------------------------------- + * Function: check_objects + * + * Purpose: locate all HDF5 objects in the file and compare with user + * supplied list + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ +int check_objects(const char* fname, + pack_opt_t *options) +{ + hid_t fid; + int i; + trav_table_t *travt=NULL; + +/*------------------------------------------------------------------------- + * open the file + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY { + + /* Open the files */ + if ((fid=H5Fopen(fname,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ){ + printf("h5repack: <%s>: %s\n", fname, H5FOPENERROR ); + exit(1); + } + /* enable error reporting */ + } H5E_END_TRY; + + +/*------------------------------------------------------------------------- + * get the list of objects in the file + *------------------------------------------------------------------------- + */ + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fid,travt)<0) + goto out; + +/*------------------------------------------------------------------------- + * compare with user supplied list + *------------------------------------------------------------------------- + */ + + if (options->verbose) + { + printf("\n"); + printf("Opening file <%s>. Searching for objects to modify...\n",fname); + } + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + if (options->verbose) + printf(PFORMAT1,"","",name); + + /* the input object names are present in the file and are valid */ + if (h5trav_getindext(name,travt)<0) + { + printf("\nError: Could not find <%s> in file <%s>. Exiting...\n", + name,fname); + goto out; + } + if (options->verbose) + printf("...Found\n"); + } +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + H5Fclose(fid); + trav_table_free(travt); + return 0; + +out: + H5Fclose(fid); + trav_table_free(travt); + return -1; +} + + + +/*------------------------------------------------------------------------- + * Function: print_objlist + * + * Purpose: print list of objects in file + * + * Return: void + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October 23, 2003 + * + *------------------------------------------------------------------------- + */ +void print_objlist(const char *filename, + int nobjects, + trav_info_t *info ) +{ + int i; + + printf("File <%s>: # of entries = %d\n", filename, nobjects ); + for ( i = 0; i < nobjects; i++) + { + switch ( info[i].type ) + { + case H5G_GROUP: + printf(" %-10s %s\n", "group", info[i].name ); + break; + case H5G_DATASET: + printf(" %-10s %s\n", "dataset", info[i].name ); + break; + case H5G_TYPE: + printf(" %-10s %s\n", "datatype", info[i].name ); + break; + case H5G_LINK: + printf(" %-10s %s\n", "link", info[i].name ); + break; + default: + printf(" %-10s %s\n", "User defined object", info[i].name ); + break; + } + } + +} + diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c new file mode 100644 index 0000000..cc3f6ea --- /dev/null +++ b/tools/h5repack/h5repack_main.c @@ -0,0 +1,148 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include "h5repack.h" + +static void usage(void); + +/* +h5repack main program +*/ + +int main(int argc, char **argv) +{ + char *infile = NULL; + char *outfile = NULL; + pack_opt_t options; /*the global options */ + int i, ret; + + /* initialize options */ + h5repack_init (&options,0); + + for ( i = 1; i < argc; i++) + { + if (strcmp(argv[i], "-i") == 0) { + infile = argv[++i]; + } + else if (strcmp(argv[i], "-o") == 0) { + outfile = argv[++i]; + } + else if (strcmp(argv[i], "-v") == 0) { + options.verbose = 1; + } + else if (strcmp(argv[i], "-f") == 0) { + + /* add the -f filter option */ + h5repack_addfilter(argv[i+1],&options); + + /* jump to next */ + ++i; + } + else if (strcmp(argv[i], "-l") == 0) { + + /* parse the -l layout option */ + h5repack_addlayout(argv[i+1],&options); + + /* jump to next */ + ++i; + } + + else if (strcmp(argv[i], "-m") == 0) { + options.threshold = parse_number(argv[i+1]); + if ((int)options.threshold==-1) { + printf("Error: Invalid treshold size <%s>\n",argv[i+1]); + exit(1); + } + ++i; + } + + else if (strcmp(argv[i], "-e") == 0) { + read_info(argv[++i],&options); + } + + else if (argv[i][0] == '-') { + usage(); + } + } + + if (infile == NULL || outfile == NULL) + usage(); + + /* pack it */ + ret=h5repack(infile,outfile,&options); + + /* free tables */ + h5repack_end(&options); + + if (ret==-1) + return 1; + else + return 0; +} + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: print usage + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +static +void usage(void) +{ + printf("h5repack -i input -o output [-h] [-v] [-f 'comp_info'] [-l 'chunk_info'][-m number][-e file] \n"); + printf("\n"); + printf("-i input Input HDF5 File\n"); + printf("-o output Output HDF5 File\n"); + printf("[-h] Print usage message\n"); + printf("[-f 'filter'] Filter type: 'filter' is a string with the format\n"); + printf(" <list of objects> : <name of filter> <filter parameters>\n"); + printf(" <list of objects> is a comma separated list of object names\n"); + printf(" meaning apply compression only to those objects.\n"); + printf(" if no object names are specified, the filter is applied to all objects\n"); + printf(" <name of filter> can be:\n"); + printf(" GZIP, to apply the HDF5 GZIP filter (GZIP compression)\n"); + printf(" SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n"); + printf(" SHUF, to apply the HDF5 shuffle filter\n"); + printf(" FLET, to apply the HDF5 checksum filter\n"); + printf(" NONE, to remove the filter\n"); + printf(" <filter parameters> is optional compression info\n"); + printf(" GZIP, the deflation level\n"); + printf(" SZIP, the pixels per block parameter\n"); + printf("[-l 'layout'] Layout type. 'layout' is a string with the format\n"); + printf(" <list of objects> : <layout type>\n"); + printf(" <list of objects> is a comma separated list of object names,\n"); + printf(" meaning that layout information is supplied for those objects.\n"); + printf(" if no object names are specified, the layout is applied to all objects\n"); + printf(" <layout type> can be:\n"); + printf(" CHUNK, to apply chunking layout\n"); + printf(" COMPA, to apply compact layout\n"); + printf(" CONTI, to apply continuous layout\n"); + printf(" <layout parameters> is present for the chunk case only\n"); + printf(" it is the chunk size of each dimension:\n"); + printf(" <dim_1 x dim_2 x ... dim_n>\n"); + printf("\n"); + printf("-e file File with the -f and -l options (only filter and layout flags)\n"); + printf("-m number Do not apply the filter to objects which size in bytes is smaller than number.\n"); + printf(" If no size is specified a minimum of 1024 bytes is assumed.\n"); + printf("\n"); + + exit(1); +} + + diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c new file mode 100644 index 0000000..45dfac5 --- /dev/null +++ b/tools/h5repack/h5repack_opttable.c @@ -0,0 +1,377 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include <stdlib.h> +#include "h5repack.h" + + + +/*------------------------------------------------------------------------- + * Function: init_packobject + * + * Purpose: initialize a pack_info_t structure + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +void init_packobject(pack_info_t *obj) +{ + int j, k; + + strcpy(obj->path,"\0"); + for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) + { + obj->filter[j].filtn = -1; + for ( k=0; k<CDVALUES; k++) + obj->filter[j].cd_values[k] = -1; + } + obj->chunk.rank = -1; + obj->refobj_id = -1; + obj->layout = H5D_LAYOUT_ERROR; + obj->nfilters = 0; + + +} + +/*------------------------------------------------------------------------- + * Function: aux_tblinsert_filter + * + * Purpose: auxiliary function, inserts the filter in object OBJS[ I ] + * + * Return: void + * + *------------------------------------------------------------------------- + */ + +static void aux_tblinsert_filter(pack_opttbl_t *table, + int I, + filter_info_t filt) +{ + if (table->objs[ I ].nfilters<H5_REPACK_MAX_NFILTERS) + { + table->objs[ I ].filter[ table->objs[ I ].nfilters++ ] = filt; + } + else + { + printf("Cannot insert the filter in this object.\ + Maximum capacity exceeded\n"); + } +} + + +/*------------------------------------------------------------------------- + * Function: aux_inctable + * + * Purpose: auxiliary function, increases the size of the collection by N_OBJS + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +static int aux_inctable(pack_opttbl_t *table, int n_objs ) +{ + int i, j, k; + + table->size += n_objs; + table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + for (i = table->nelems; i < table->size; i++) + { + strcpy(table->objs[i].path,"\0"); + for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) + { + table->objs[i].filter[j].filtn = -1; + for ( k=0; k<CDVALUES; k++) + table->objs[i].filter[j].cd_values[k] = -1; + } + table->objs[i].chunk.rank = -1; + table->objs[i].refobj_id = -1; + table->objs[i].layout = H5D_LAYOUT_ERROR; + table->objs[i].nfilters = 0; + } + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_table_init + * + * Purpose: init options table + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int options_table_init( pack_opttbl_t **tbl ) +{ + int i, j, k; + pack_opttbl_t* table = (pack_opttbl_t*) malloc(sizeof(pack_opttbl_t)); + if (table==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + + table->size = 30; + table->nelems = 0; + table->objs = (pack_info_t*) malloc(table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + + for ( i=0; i<table->size; i++) + { + strcpy(table->objs[i].path,"\0"); + for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) + { + table->objs[i].filter[j].filtn = -1; + for ( k=0; k<CDVALUES; k++) + table->objs[i].filter[j].cd_values[k] = -1; + } + table->objs[i].chunk.rank = -1; + table->objs[i].refobj_id = -1; + table->objs[i].layout = H5D_LAYOUT_ERROR; + table->objs[i].nfilters = 0; + } + + *tbl = table; + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_table_free + * + * Purpose: free table memory + * + * Return: 0 + * + *------------------------------------------------------------------------- + */ + +int options_table_free( pack_opttbl_t *table ) +{ + free(table->objs); + free(table); + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_add_layout + * + * Purpose: add a layout option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + + +int options_add_layout( obj_list_t *obj_list, + int n_objs, + pack_info_t *pack, + pack_opttbl_t *table ) +{ + int i, j, k, I, added=0, found=0; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems+n_objs >= table->size) + { + if (aux_inctable(table,n_objs)<0) + return -1; + } + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* already chunk info inserted for this one; exit */ + if (table->objs[i].chunk.rank>0) + { + printf("Input Error: chunk information already inserted for <%s>\n",obj_list[j].obj); + exit(1); + } + /* insert the layout info */ + else + { + table->objs[i].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[i].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[i].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } + found=1; + break; + } + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } + } + } + + table->nelems+= added; + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: options_add_filter + * + * Purpose: add a compression -f option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int options_add_filter(obj_list_t *obj_list, + int n_objs, + filter_info_t filt, + pack_opttbl_t *table ) +{ + + int i, j, I, added=0, found=0; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems+n_objs >= table->size) + { + if (aux_inctable(table,n_objs)<0) + return -1; + } + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* insert */ + aux_tblinsert_filter(table,i,filt); + found=1; + break; + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_filter(table,I,filt); + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_filter(table,I,filt); + } + } + + table->nelems+= added; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_get_object + * + * Purpose: get object from table; "path" is the key + * + * Return: pack_info_t* OBJECT or NULL if not found; PATH is the key + * + *------------------------------------------------------------------------- + */ + +pack_info_t* options_get_object( const char *path, + pack_opttbl_t *table ) +{ + int i; + + for ( i = 0; i < table->nelems; i++) + { + /* found it */ + if (strcmp(table->objs[i].path,path)==0) + { + return (&table->objs[i]); + } + } + + return NULL; +} + + + + diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c new file mode 100644 index 0000000..204adf3 --- /dev/null +++ b/tools/h5repack/h5repack_parse.c @@ -0,0 +1,508 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <ctype.h> +#include "h5repack.h" + +#if 0 +#define PARSE_DEBUG +#endif + +/*------------------------------------------------------------------------- + * Function: parse_filter + * + * Purpose: read filter information + * + * Return: a list of names, the number of names and its compression type + * + * <name of filter> can be: + * GZIP, to apply the HDF5 GZIP filter (GZIP compression) + * SZIP, to apply the HDF5 SZIP filter (SZIP compression) + * SHUF, to apply the HDF5 shuffle filter + * FLET, to apply the HDF5 checksum filter + * NONE, to remove the filter + * + * Examples: + * "GZIP=6" + * "A,B:NONE" + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +obj_list_t* parse_filter(const char *str, + int *n_objs, + filter_info_t *filt, + pack_opt_t *options) +{ + unsigned i, u; + char c; + size_t len=strlen(str); + int j, m, n, k, end_obj=-1, no_param=0; + char sobj[MAX_NC_NAME]; + char scomp[10]; + char stype[5]; + obj_list_t* obj_list=NULL; + unsigned pixels_per_block; + +#if defined(PARSE_DEBUG) + fprintf(stderr,"%s\n",str); +#endif + + /* initialize compression info */ + memset(filt,0,sizeof(filter_info_t)); + + /* check for the end of object list and number of objects */ + for ( i=0, n=0; i<len; i++) + { + c = str[i]; + if ( c==':' ) + { + end_obj=i; + } + if ( c==',' ) + { + n++; + } + } + + if (end_obj==-1) { /* missing : */ + /* apply to all objects */ + options->all_filter=1; + } + + n++; + obj_list=malloc(n*sizeof(obj_list_t)); + if (obj_list==NULL) + { + printf("Could not alloc object list\n"); + return NULL; + } + *n_objs=n; + + /* get object list */ + for ( j=0, k=0, n=0; j<end_obj; j++,k++) + { + c = str[j]; + sobj[k]=c; + if ( c==',' || j==end_obj-1) + { + if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; + strcpy(obj_list[n].obj,sobj); + memset(sobj,0,sizeof(sobj)); + n++; + k=-1; + } + } + /* nothing after : */ + if (end_obj+1==(int)len) + { + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression type in <%s>\n",str); + exit(1); + } + + + /* get filter additional parameters */ + m=0; + for ( i=end_obj+1, k=0, j=0; i<len; i++,k++) + { + c = str[i]; + scomp[k]=c; + if ( c=='=' || i==len-1) + { + if ( c=='=') { /*one more parameter */ + scomp[k]='\0'; /*cut space */ + + /* here we could have 1, 2 or 3 digits */ + for ( m=0,u=i+1; u<len; u++,m++) { + c = str[u]; + if (!isdigit(c)){ + if (obj_list) free(obj_list); + printf("Input Error: Compression parameter not digit in <%s>\n",str); + exit(1); + } + stype[m]=c; + } + stype[m]='\0'; + filt->cd_values[j++]=atoi(stype); + i+=m; /* jump */ + } + else if (i==len-1) { /*no more parameters */ + scomp[k+1]='\0'; + no_param=1; + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_NONE + *------------------------------------------------------------------------- + */ + if (strcmp(scomp,"NONE")==0) + filt->filtn=H5Z_FILTER_NONE; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"GZIP")==0) + { + filt->filtn=H5Z_FILTER_DEFLATE; + if (no_param) { /*no more parameters, GZIP must have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Missing compression parameter in <%s>\n",str); + exit(1); + } + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SZIP")==0) + { + filt->filtn=H5Z_FILTER_SZIP; + if (no_param) { /*no more parameters, SZIP must have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Missing compression parameter in <%s>\n",str); + exit(1); + } + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SHUF")==0) + { + filt->filtn=H5Z_FILTER_SHUFFLE; + if (m>0){ /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Extra parameter in SHUF <%s>\n",str); + exit(1); + } + } +/*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"FLET")==0) + { + filt->filtn=H5Z_FILTER_FLETCHER32; + if (m>0){ /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Extra parameter in FLET <%s>\n",str); + exit(1); + } + } + else { + if (obj_list) free(obj_list); + printf("Input Error: Invalid filter type in <%s>\n",str); + exit(1); + } + } + } /*i*/ + +/*------------------------------------------------------------------------- + * check valid parameters + *------------------------------------------------------------------------- + */ + + switch (filt->filtn) + { + + case H5Z_FILTER_DEFLATE: + if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ){ + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + + + case H5Z_FILTER_SZIP: + pixels_per_block=filt->cd_values[0]; + if ((pixels_per_block%2)==1) { + if (obj_list) free(obj_list); + printf("Input Error: pixels_per_block is not even in <%s>\n",str); + exit(1); + } + if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) { + if (obj_list) free(obj_list); + printf("Input Error: pixels_per_block is too large in <%s>\n",str); + exit(1); + } + break; + }; + + return obj_list; +} + + +/*------------------------------------------------------------------------- + * Function: get_sfilter + * + * Purpose: return the filter as a string name + * + * Return: name of filter, exit on error + * + *------------------------------------------------------------------------- + */ + +const char* get_sfilter(H5Z_filter_t filtn) +{ + if (filtn==H5Z_FILTER_NONE) + return "NONE"; + else if (filtn==H5Z_FILTER_DEFLATE) + return "GZIP"; + else if (filtn==H5Z_FILTER_SZIP) + return "SZIP"; + else if (filtn==H5Z_FILTER_SHUFFLE) + return "SHUFFLE"; + else if (filtn==H5Z_FILTER_FLETCHER32) + return "FLETCHERP"; + else { + printf("Input error in filter type\n"); + exit(1); + } + return NULL; +} + + +/*------------------------------------------------------------------------- + * Function: parse_layout + * + * Purpose: read layout info + * + * Return: a list of names, the number of names and its chunking info for + * chunked. NULL, on error + * the layout type can be: + * CHUNK, to apply chunking layout + * CONTI, to apply continuous layout + * COMPA, to apply compact layout + * + * Example: + * "AA,B,CDE:CHUNK=10X10" + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ +obj_list_t* parse_layout(const char *str, + int *n_objs, + pack_info_t *pack, /* info about layout needed */ + pack_opt_t *options) +{ + obj_list_t* obj_list=NULL; + unsigned i; + char c; + size_t len=strlen(str); + int j, n, k, end_obj=-1, c_index; + char sobj[MAX_NC_NAME]; + char sdim[10]; + char slayout[10]; + + /* check for the end of object list and number of objects */ + for ( i=0, n=0; i<len; i++) + { + c = str[i]; + if ( c==':' ) + { + end_obj=i; + } + if ( c==',' ) + { + n++; + } + } + + if (end_obj==-1) { /* missing : chunk all */ + options->all_layout=1; + } + + n++; + obj_list=malloc(n*sizeof(obj_list_t)); + if (obj_list==NULL) + { + printf("Could not alloc object list\n"); + return NULL; + } + *n_objs=n; + + /* get object list */ + for ( j=0, k=0, n=0; j<end_obj; j++,k++) + { + c = str[j]; + sobj[k]=c; + if ( c==',' || j==end_obj-1) + { + if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; + strcpy(obj_list[n].obj,sobj); + memset(sobj,0,sizeof(sobj)); + n++; + k=-1; + } + } + + /* nothing after : */ + if (end_obj+1==(int)len) + { + if (obj_list) free(obj_list); + printf("Parse layout error: No characters after : in <%s>\n",str); + exit(1); + } + + /* get layout info */ + for ( j=end_obj+1, n=0; n<=5; j++,n++) + { + if (n==5) + { + slayout[n]='\0'; /*cut string */ + if (strcmp(slayout,"COMPA")==0) + pack->layout=H5D_COMPACT; + else if (strcmp(slayout,"CONTI")==0) + pack->layout=H5D_CONTIGUOUS; + else if (strcmp(slayout,"CHUNK")==0) + pack->layout=H5D_CHUNKED; + else { + printf("Parse layout error: Not a valid layout in <%s>\n",str); + } + } + else + { + c = str[j]; + slayout[n]=c; + } + } /* j */ + + + if ( pack->layout==H5D_CHUNKED ) + { + +/*------------------------------------------------------------------------- + * get chunk info + *------------------------------------------------------------------------- + */ + k=0; + + if (j>(int)len) + { + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Chunk dimensions missing\n",str); + exit(1); + } + + for ( i=j, c_index=0; i<len; i++) + { + c = str[i]; + sdim[k]=c; + k++; /*increment sdim index */ + + if (!isdigit(c) && c!='x' && c!='N' && c!='O' && c!='N' && c!='E'){ + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Not a valid character in <%s>\n", + sdim,str); + exit(1); + } + + if ( c=='x' || i==len-1) + { + if ( c=='x') { + sdim[k-1]='\0'; + k=0; + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0) { + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Conversion to number in <%s>\n", + sdim,str); + exit(1); + } + c_index++; + } + else if (i==len-1) { /*no more parameters */ + sdim[k]='\0'; + k=0; + if (strcmp(sdim,"NONE")==0) + { + pack->chunk.rank=-2; + } + else + { + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0){ + if (obj_list) free(obj_list); + printf("Parse layout error: <%s> Conversion to number in <%s>\n", + sdim,str); + exit(1); + } + pack->chunk.rank=c_index+1; + } + } /*if */ + } /*if c=='x' || i==len-1 */ + } /*i*/ + + + } /*H5D_CHUNKED*/ + + + return obj_list; +} + + + +/*------------------------------------------------------------------------- + * Function: parse_number + * + * Purpose: read a number from command line argument + * + * Return: number, -1 for FAIL + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +int parse_number(char *str) +{ + unsigned i; + int n; + char c; + size_t len=strlen(str); + + for ( i=0; i<len; i++) + { + c = str[i]; + if (!isdigit(c)){ + return -1; + } + } + str[i]='\0'; + n=atoi(str); + return n; +} + + + + diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c new file mode 100644 index 0000000..e824493 --- /dev/null +++ b/tools/h5repack/h5repack_refs.c @@ -0,0 +1,786 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include "H5private.h" +#include "h5repack.h" + +static const char* MapIdToName(hid_t refobj_id, + trav_table_t *travt); + +static void close_obj(H5G_obj_t obj_type, hid_t obj_id); + + +static int copy_refs_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options, + trav_table_t *travt, + hid_t fidout /* for saving references */ + ); + +/*------------------------------------------------------------------------- + * Function: do_copy_refobjs + * + * Purpose: duplicate all referenced HDF5 objects in the file + * and create hard links + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December, 10, 2003 + * + *------------------------------------------------------------------------- + */ + +int do_copy_refobjs(hid_t fidin, + hid_t fidout, + trav_table_t *travt, + pack_opt_t *options) /* repack options */ +{ + hid_t grp_in=-1; /* group ID */ + hid_t grp_out=-1; /* group ID */ + hid_t dset_in=-1; /* read dataset ID */ + hid_t dset_out=-1; /* write dataset ID */ + hid_t type_in=-1; /* named type ID */ + hid_t dcpl_id=-1; /* dataset creation property list ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file data type ID */ + hid_t mtype_id=-1; /* memory data type ID */ + size_t msize; /* memory size of memory type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + int i, j; + +/*------------------------------------------------------------------------- + * browse + *------------------------------------------------------------------------- + */ + + for ( i = 0; i < travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + /*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + case H5G_GROUP: + + /*------------------------------------------------------------------------- + * check for hard links + *------------------------------------------------------------------------- + */ + + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++) + { + H5Glink(fidout, + H5G_LINK_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name); + } + } + + break; + + /*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + case H5G_DATASET: + + if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) + goto error; + if ((space_id=H5Dget_space(dset_in))<0) + goto error; + if ((ftype_id=H5Dget_type (dset_in))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_in))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) + goto error; + if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) + goto error; + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0) + goto error; + if ((msize=H5Tget_size(mtype_id))==0) + goto error; + +/*------------------------------------------------------------------------- + * check if the dataset creation property list has filters that + * are not registered in the current configuration + * 1) the external filters GZIP and SZIP might not be available + * 2) the internal filters might be turned off + *------------------------------------------------------------------------- + */ + if (h5tools_canreadf((options->verbose?travt->objs[i].name:NULL),dcpl_id)==1) + { + +/*------------------------------------------------------------------------- + * test for a valid output dataset + *------------------------------------------------------------------------- + */ + dset_out = FAIL; + +/*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + *------------------------------------------------------------------------- + */ + if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hobj_ref_t *refbuf=NULL; /* buffer for object references */ + hobj_ref_t *buf=NULL; + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + + if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) + goto error; + refbuf=HDmalloc((unsigned)nelmts*msize); + if ( refbuf==NULL){ + printf( "cannot allocate memory\n" ); + goto error; + } + for ( u=0; u<nelmts; u++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(dset_in,H5R_OBJECT,&buf[u]))<0) + continue; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + /* create the reference, -1 parameter for objects */ + if (H5Rcreate(&refbuf[u],fidout,refname,H5R_OBJECT,-1)<0) + goto error; + if (options->verbose) + printf("object <%s> object reference created to <%s>\n", + travt->objs[i].name, + refname); + }/*refname*/ + close_obj(obj_type,refobj_id); + }/* u */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + goto error; + if (nelmts) { + if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + goto error; + } + + if (buf) + free(buf); + if (refbuf) + free(refbuf); + + }/*H5T_STD_REF_OBJ*/ + +/*------------------------------------------------------------------------- + * dataset region references + *------------------------------------------------------------------------- + */ + else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ + hdset_reg_ref_t *buf=NULL; /* output buffer */ + const char* refname; + unsigned u; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) + goto error; + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + + refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ + if ( refbuf==NULL){ + printf( "cannot allocate memory\n" ); + goto error; + } + for ( u=0; u<nelmts; u++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(dset_in,H5R_DATASET_REGION,&buf[u]))<0) + continue; + } H5E_END_TRY; + + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + hid_t region_id; /* region id of the referenced dataset */ + if ((region_id = H5Rget_region(dset_in,H5R_DATASET_REGION,&buf[u]))<0) + goto error; + /* create the reference, we need the space_id */ + if (H5Rcreate(&refbuf[u],fidout,refname,H5R_DATASET_REGION,region_id)<0) + goto error; + if (H5Sclose(region_id)<0) + goto error; + if (options->verbose) + printf("object <%s> region reference created to <%s>\n", + travt->objs[i].name, + refname); + }/*refname*/ + close_obj(obj_type,refobj_id); + }/* u */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * create/write dataset/close + *------------------------------------------------------------------------- + */ + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + goto error; + if (nelmts) { + if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + goto error; + } + + if (buf) + free(buf); + if (refbuf) + free(refbuf); + } /* H5T_STD_REF_DSETREG */ + + +/*------------------------------------------------------------------------- + * not references, open previously created object in 1st traversal + *------------------------------------------------------------------------- + */ + else + { + if ((dset_out=H5Dopen(fidout,travt->objs[i].name))<0) + goto error; + } + + assert(dset_out!=FAIL); + +/*------------------------------------------------------------------------- + * copy referenced objects in attributes + *------------------------------------------------------------------------- + */ + if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0) + goto error; + + +/*------------------------------------------------------------------------- + * check for hard links + *------------------------------------------------------------------------- + */ + if (travt->objs[i].nlinks) + { + for ( j=0; j<travt->objs[i].nlinks; j++){ + H5Glink(fidout, + H5G_LINK_HARD, + travt->objs[i].name, + travt->objs[i].links[j].new_name); + } + } + + if (H5Dclose(dset_out)<0) + goto error; + + }/*can_read*/ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id)<0) + goto error; + if (H5Tclose(mtype_id)<0) + goto error; + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_in)<0) + goto error; + + break; + + /*------------------------------------------------------------------------- + * H5G_TYPE + *------------------------------------------------------------------------- + */ + case H5G_TYPE: + + if ((type_in = H5Topen (fidin,travt->objs[i].name))<0) + goto error; + + if (H5Tclose(type_in)<0) + goto error; + + break; + + /*------------------------------------------------------------------------- + * H5G_LINK + *------------------------------------------------------------------------- + */ + + case H5G_LINK: + + /*nothing to do */ + break; + + default: + + break; + } + } + + + +/*------------------------------------------------------------------------- + * the root is a special case, we get an ID for the root group + * and copy its attributes using that ID + * it must be done last, because the attributes might contain references to + * objects in the object list + *------------------------------------------------------------------------- + */ + + if ((grp_out = H5Gopen(fidout,"/"))<0) + goto error; + + if ((grp_in = H5Gopen(fidin,"/"))<0) + goto error; + + if (copy_refs_attr(grp_in,grp_out,options,travt,fidout)<0) + goto error; + + if (H5Gclose(grp_out)<0) + goto error; + if (H5Gclose(grp_in)<0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Gclose(grp_in); + H5Gclose(grp_out); + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_in); + H5Dclose(dset_out); + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Tclose(type_in); + } H5E_END_TRY; + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: copy_refs_attr + * + * Purpose: duplicate all referenced HDF5 located in attributes + * relative to LOC_IN, which is obtained either from + * loc_id = H5Gopen( fid, name); + * loc_id = H5Dopen( fid, name); + * loc_id = H5Topen( fid, name); + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: October, 28, 2003 + * + *------------------------------------------------------------------------- + */ + +static int copy_refs_attr(hid_t loc_in, + hid_t loc_out, + pack_opt_t *options, + trav_table_t *travt, + hid_t fidout /* for saving references */ + ) +{ + hid_t attr_id=-1; /* attr ID */ + hid_t attr_out=-1; /* attr ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file data type ID */ + hid_t mtype_id=-1; /* memory data type ID */ + size_t msize; /* memory size of type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + int n, j; + unsigned u; + + if ((n = H5Aget_num_attrs(loc_in))<0) + goto error; + + for ( u = 0; u < (unsigned)n; u++) + { + +/*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + /* open attribute */ + if ((attr_id = H5Aopen_idx(loc_in, u))<0) + goto error; + + /* get name */ + if (H5Aget_name( attr_id, 255, name )<0) + goto error; + + /* get the file datatype */ + if ((ftype_id = H5Aget_type( attr_id )) < 0 ) + goto error; + + /* get the dataspace handle */ + if ((space_id = H5Aget_space( attr_id )) < 0 ) + goto error; + + /* get dimensions */ + if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) + goto error; + + + /*------------------------------------------------------------------------- + * elements + *------------------------------------------------------------------------- + */ + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0) + goto error; + if ((msize=H5Tget_size(mtype_id))==0) + goto error; + +/*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + *------------------------------------------------------------------------- + */ + if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hobj_ref_t *refbuf=NULL; + unsigned k; + const char* refname; + hobj_ref_t *buf=NULL; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Aread(attr_id,mtype_id,buf)<0) + goto error; + + if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0) + goto error; + refbuf=HDmalloc((unsigned)nelmts*msize); + if ( refbuf==NULL){ + printf( "cannot allocate memory\n" ); + goto error; + } + for ( k=0; k<nelmts; k++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(attr_id,H5R_OBJECT,&buf[k]))<0) + goto error; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + /* create the reference */ + if (H5Rcreate(&refbuf[k],fidout,refname,H5R_OBJECT,-1)<0) + goto error; + if (options->verbose) + printf("object <%s> reference created to <%s>\n",name,refname); + } + close_obj(obj_type,refobj_id); + }/* k */ + }/*nelmts*/ + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) + goto error; + if (nelmts) { + if(H5Awrite(attr_out,mtype_id,refbuf)<0) + goto error; + } + + if (H5Aclose(attr_out)<0) + goto error; + + if (refbuf) + free(refbuf); + if (buf) + free(buf); + + }/*H5T_STD_REF_OBJ*/ + +/*------------------------------------------------------------------------- + * dataset region references + *------------------------------------------------------------------------- + */ + else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + { + H5G_obj_t obj_type; + hid_t refobj_id; + hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ + hdset_reg_ref_t *buf=NULL; /* output buffer */ + const char* refname; + unsigned k; + + /*------------------------------------------------------------------------- + * read input to memory + *------------------------------------------------------------------------- + */ + + if (nelmts) + { + buf=(void *) HDmalloc((unsigned)(nelmts*msize)); + if ( buf==NULL){ + printf( "cannot read into memory\n" ); + goto error; + } + if (H5Aread(attr_id,mtype_id,buf)<0) + goto error; + if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0) + goto error; + + /*------------------------------------------------------------------------- + * create output + *------------------------------------------------------------------------- + */ + + refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ + if ( refbuf==NULL){ + printf( "cannot allocate memory\n" ); + goto error; + } + for ( k=0; k<nelmts; k++) + { + H5E_BEGIN_TRY { + if ((refobj_id = H5Rdereference(attr_id,H5R_DATASET_REGION,&buf[k]))<0) + continue; + } H5E_END_TRY; + /* get the name. a valid name could only occur in the + second traversal of the file */ + if ((refname=MapIdToName(refobj_id,travt))!=NULL) + { + hid_t region_id; /* region id of the referenced dataset */ + if ((region_id = H5Rget_region(attr_id,H5R_DATASET_REGION,&buf[k]))<0) + goto error; + /* create the reference, we need the space_id */ + if (H5Rcreate(&refbuf[k],fidout,refname,H5R_DATASET_REGION,region_id)<0) + goto error; + if (H5Sclose(region_id)<0) + goto error; + if (options->verbose) + printf("object <%s> region reference created to <%s>\n",name,refname); + } + close_obj(obj_type,refobj_id); + }/* k */ + }/*nelmts */ + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) + goto error; + if (nelmts) { + if(H5Awrite(attr_out,mtype_id,refbuf)<0) + goto error; + } + if (H5Aclose(attr_out)<0) + goto error; + if (refbuf) + free(refbuf); + if (buf) + free(buf); + } /* H5T_STD_REF_DSETREG */ + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id)<0) goto error; + if (H5Tclose(mtype_id)<0) goto error; + if (H5Sclose(space_id)<0) goto error; + if (H5Aclose(attr_id)<0) goto error; + } /* u */ + + return 0; + +error: + H5E_BEGIN_TRY { + H5Tclose(ftype_id); + H5Tclose(mtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + } H5E_END_TRY; + return -1; +} + +/*------------------------------------------------------------------------- + * Function: close_obj + * + * Purpose: Auxiliary function to close an object + * + *------------------------------------------------------------------------- + */ + +static void close_obj(H5G_obj_t obj_type, hid_t obj_id) +{ + H5E_BEGIN_TRY + { + switch (obj_type) + { + case H5G_GROUP: + H5Gclose(obj_id); + break; + case H5G_DATASET: + H5Dclose(obj_id); + break; + case H5G_TYPE: + H5Tclose(obj_id); + break; + default: + break; + } + } H5E_END_TRY; +} + +/*------------------------------------------------------------------------- + * Function: MapIdToName + * + * Purpose: map an object ID to a name + * + *------------------------------------------------------------------------- + */ + +static const char* MapIdToName(hid_t refobj_id, + trav_table_t *travt) +{ + hid_t id; + hid_t fid; + int i; + + /* obtains the file ID given an object ID. This ID must be closed */ + if ((fid = H5Iget_file_id(refobj_id))<0) + { + return NULL; + } + + /* linear search */ + for ( i=0; i<travt->nobjs; i++) + { + switch ( travt->objs[i].type ) + { + default: + break; + + /*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + + case H5G_DATASET: + + if ((id = H5Dopen(fid,travt->objs[i].name))<0) + return NULL; + if (H5Dclose(id)<0) + return NULL; + if (id==refobj_id) + { + H5Fclose(fid); + return travt->objs[i].name; + } + break; + } /* switch */ + } /* i */ + + if (H5Fclose(fid)<0) + return NULL; + + return NULL; +} + diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c new file mode 100644 index 0000000..4f69d5f --- /dev/null +++ b/tools/h5repack/h5repack_verify.c @@ -0,0 +1,487 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + + +/*------------------------------------------------------------------------- + * Function: has_filter + * + * Purpose: verify if a filter is present in the property list DCPL_ID + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int has_filter(hid_t dcpl_id, + H5Z_filter_t filtnin) +{ + + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + int have=0; /* flag, filter is present */ + int i; /* index */ + + /* if no information about the input filter is requested return exit */ + if (filtnin==-1) + return 1; + + /* get information about filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + /* if we do not have filters and the requested filter is NONE, return 1 */ + if (!nfilters && filtnin==H5Z_FILTER_NONE) + return 1; + + for (i=0; i<nfilters; i++) + { + cd_nelmts = NELMTS(cd_values); + filtn = H5Pget_filter(dcpl_id, + (unsigned)i, + &filt_flags, + &cd_nelmts, + cd_values, + sizeof(f_name), + f_name); + + if (filtnin==filtn) + have=1; + + } + + return have; +} + + +/*------------------------------------------------------------------------- + * Function: has_layout + * + * Purpose: verify which layout is present in the property list DCPL_ID + * + * H5D_COMPACT = 0 + * H5D_CONTIGUOUS = 1 + * H5D_CHUNKED = 2 + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ + +int has_layout(hid_t dcpl_id, + pack_info_t *obj) +{ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; /* layout */ + int nfilters; /* number of filters */ + int rank; /* rank */ + int i; /* index */ + + /* if no information about the input layout is requested return exit */ + if (obj==NULL) + return 1; + + /* check if we have filters in the input object */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + /* a non chunked layout was requested on a filtered object; avoid the test */ + if (nfilters && obj->layout!=H5D_CHUNKED) + return 1; + + /* get layout */ + if ((layout = H5Pget_layout(dcpl_id))<0) + return -1; + + if (obj->layout != layout) + return 0; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + if (obj->chunk.rank != rank) + return 0; + for ( i=0; i<rank; i++) + if (chsize[i] != obj->chunk.chunk_lengths[i]) + return 0; + } + + return 1; +} + + +/*------------------------------------------------------------------------- + * Function: h5repack_verify + * + * Purpose: verify if the filters specified in the options list are + * present on the OUTPUT file + * + * Return: 1=filter present, 0=filter not present, -1=error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int h5repack_verify(const char *fname, + pack_opt_t *options) +{ + hid_t fid; /* file ID */ + hid_t dset_id=-1; /* dataset ID */ + hid_t dcpl_id=-1; /* dataset creation property list ID */ + hid_t space_id=-1; /* space ID */ + int ret=1, i, j; + trav_table_t *travt=NULL; + + /* open the file */ + if ((fid=H5Fopen(fname,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ) + return -1; + + for ( i=0; i<options->op_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + pack_info_t *obj = &options->op_tbl->objs[i]; + +/*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if ((dset_id=H5Dopen(fid,name))<0) + goto error; + if ((space_id=H5Dget_space(dset_id))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_id))<0) + goto error; + + if (options->verbose) { + printf(" %-10s %s\n", "dataset",name ); + print_filters(dcpl_id); + } + +/*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + for ( j=0; j<obj->nfilters; j++) + { + if (has_filter(dcpl_id,obj->filter[j].filtn)==0) + ret=0; + } + +/*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if ((obj->layout!=-1) && (has_layout(dcpl_id,obj)==0)) + ret=0; + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_id)<0) + goto error; + + } + + +/*------------------------------------------------------------------------- + * check for the "all" objects option + *------------------------------------------------------------------------- + */ + + if (options->all_filter==1 || options->all_layout==1) + { + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fid,travt)<0) + goto error; + + for ( i=0; i<travt->nobjs; i++) + { + char* name=travt->objs[i].name; + + switch ( travt->objs[i].type ) + { + case H5G_DATASET: + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if ((dset_id=H5Dopen(fid,name))<0) + goto error; + if ((space_id=H5Dget_space(dset_id))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_id))<0) + goto error; + + if (options->verbose) { + printf(" %-10s %s\n", "dataset",name ); + print_filters(dcpl_id); + } + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if (options->all_filter==1 ){ + if (has_filter(dcpl_id,options->filter_g.filtn)==0) + ret=0; + } + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if (options->all_layout==1){ + pack_info_t pack; + init_packobject(&pack); + pack.layout=options->layout_g; + pack.chunk=options->chunk_g; + if (has_layout(dcpl_id,&pack)==0) + ret=0; + } + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_id)<0) + goto error; + + break; + default: + break; + } /* switch */ + + } /* i */ + + /* free table */ + trav_table_free(travt); + } + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Fclose(fid)<0) + return -1; + + return ret; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_id); + H5Fclose(fid); + if (travt) + trav_table_free(travt); + } H5E_END_TRY; + return -1; +} + + + +/*------------------------------------------------------------------------- + * Function: h5repack_cmpdcpl + * + * Purpose: compare 2 files for identical property lists of all objects + * + * Return: 1=identical, 0=not identical, -1=error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 31, 2003 + * + *------------------------------------------------------------------------- + */ + +int h5repack_cmpdcpl(const char *fname1, + const char *fname2) +{ + hid_t fid1=-1; /* file ID */ + hid_t fid2=-1; /* file ID */ + hid_t dset1=-1; /* dataset ID */ + hid_t dset2=-1; /* dataset ID */ + hid_t dcpl1=-1; /* dataset creation property list ID */ + hid_t dcpl2=-1; /* dataset creation property list ID */ + trav_table_t *travt1=NULL; + trav_table_t *travt2=NULL; + int ret=1, i; + +/*------------------------------------------------------------------------- + * open the files first; if they are not valid, no point in continuing + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY { + + /* Open the files */ + if ((fid1=H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ) + { + printf("h5repack: <%s>: %s\n", fname1, H5FOPENERROR ); + return -1; + } + if ((fid2=H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ) + { + printf("h5repack: <%s>: %s\n", fname2, H5FOPENERROR ); + H5Fclose(fid1); + return -1; + } + /* enable error reporting */ + } H5E_END_TRY; + +/*------------------------------------------------------------------------- + * get file table list of objects + *------------------------------------------------------------------------- + */ + trav_table_init(&travt1); + trav_table_init(&travt2); + if (h5trav_gettable(fid1,travt1)<0) + goto error; + if (h5trav_gettable(fid2,travt2)<0) + goto error; + + +/*------------------------------------------------------------------------- + * traverse the suppplied object list + *------------------------------------------------------------------------- + */ + + for ( i=0; i < travt1->nobjs; i++) + { + switch ( travt1->objs[i].type ) + { +/*------------------------------------------------------------------------- + * nothing to do for groups, links and types + *------------------------------------------------------------------------- + */ + default: + break; + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + case H5G_DATASET: + if ((dset1=H5Dopen(fid1,travt1->objs[i].name))<0) + goto error; + if ((dset2=H5Dopen(fid2,travt1->objs[i].name))<0) + goto error; + if ((dcpl1=H5Dget_create_plist(dset1))<0) + goto error; + if ((dcpl2=H5Dget_create_plist(dset2))<0) + goto error; + +/*------------------------------------------------------------------------- + * compare the property lists + *------------------------------------------------------------------------- + */ + if ((ret=H5Pequal(dcpl1,dcpl2))<0) + goto error; + + if (ret==0) + { + printf("Property lists for <%s> are different\n",travt1->objs[i].name); + goto error; + } + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(dcpl1)<0) + goto error; + if (H5Pclose(dcpl2)<0) + goto error; + if (H5Dclose(dset1)<0) + goto error; + if (H5Dclose(dset2)<0) + goto error; + + break; + + } /*switch*/ + } /*i*/ + +/*------------------------------------------------------------------------- + * free + *------------------------------------------------------------------------- + */ + + trav_table_free(travt1); + trav_table_free(travt2); + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + H5Fclose(fid1); + H5Fclose(fid2); + return ret; + +/*------------------------------------------------------------------------- + * error + *------------------------------------------------------------------------- + */ + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl1); + H5Pclose(dcpl2); + H5Dclose(dset1); + H5Dclose(dset2); + H5Fclose(fid1); + H5Fclose(fid2); + trav_table_free(travt1); + trav_table_free(travt2); + } H5E_END_TRY; + return -1; + +} diff --git a/tools/h5repack/testh5repack_attr.c b/tools/h5repack/testh5repack_attr.c new file mode 100644 index 0000000..a1dad92 --- /dev/null +++ b/tools/h5repack/testh5repack_attr.c @@ -0,0 +1,1032 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + +/*------------------------------------------------------------------------- + * Function: write_attr_in + * + * Purpose: write attributes in LOC_ID (dataset, group, named datatype) + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: November 12, 2003 + * + *------------------------------------------------------------------------- + */ + + +void write_attr_in(hid_t loc_id, + const char* dset_name, /* for saving reference to dataset*/ + hid_t fid, /* for reference create */ + int make_diffs /* flag to modify data buffers */) +{ + /* Compound datatype */ + typedef struct s_t + { + char a; + double b; + } s_t; + + typedef enum + { + RED, + GREEN + } e_t; + + hid_t attr_id; + hid_t space_id; + hid_t type_id; + herr_t status; + int val, i, j, k, n; + float f; + + /* create 1D attributes with dimension [2], 2 elements */ + hsize_t dims[1]={2}; + char buf1[2][2]= {"ab","de"}; /* string */ + char buf2[2]= {1,2}; /* bitfield, opaque */ + s_t buf3[2]= {{1,2},{3,4}}; /* compound */ + hobj_ref_t buf4[2]; /* reference */ + e_t buf45[2]= {RED,RED}; /* enum */ + hvl_t buf5[2]; /* vlen */ + hsize_t dimarray[1]={3}; /* array dimension */ + int buf6[2][3]= {{1,2,3},{4,5,6}}; /* array */ + int buf7[2]= {1,2}; /* integer */ + float buf8[2]= {1,2}; /* float */ + + /* create 2D attributes with dimension [3][2], 6 elements */ + hsize_t dims2[2]={3,2}; + char buf12[6][2]= {"ab","cd","ef","gh","ij","kl"}; /* string */ + char buf22[3][2]= {{1,2},{3,4},{5,6}}; /* bitfield, opaque */ + s_t buf32[6]= {{1,2},{3,4},{5,6},{7,8},{9,10},{11,12}}; /* compound */ + hobj_ref_t buf42[3][2]; /* reference */ + e_t buf452[3][2]; /* enum */ + hvl_t buf52[3][2]; /* vlen */ + int buf62[6][3]= {{1,2,3},{4,5,6},{7,8,9},{10,11,12},{13,14,15},{16,17,18}}; /* array */ + int buf72[3][2]= {{1,2},{3,4},{5,6}}; /* integer */ + float buf82[3][2]= {{1,2},{3,4},{5,6}}; /* float */ + + /* create 3D attributes with dimension [4][3][2], 24 elements */ + hsize_t dims3[3]={4,3,2}; + char buf13[24][2]= {"ab","cd","ef","gh","ij","kl","mn","pq", + "rs","tu","vw","xz","AB","CD","EF","GH", + "IJ","KL","MN","PQ","RS","TU","VW","XZ"}; /* string */ + char buf23[4][3][2]; /* bitfield, opaque */ + s_t buf33[4][3][2]; /* compound */ + hobj_ref_t buf43[4][3][2]; /* reference */ + e_t buf453[4][3][2]; /* enum */ + hvl_t buf53[4][3][2]; /* vlen */ + int buf63[24][3]; /* array */ + int buf73[4][3][2]; /* integer */ + float buf83[4][3][2]; /* float */ + + +/*------------------------------------------------------------------------- + * 1D attributes + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + for (j=0; j<2; j++) + { + buf1[i][j]='z'; + } + } + /* + buf1[2][2]= {"ab","de"}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <string> and <string> + position string of </g1> string of </g1> difference + ------------------------------------------------------------ +[ 0 ] a z +[ 0 ] b z +[ 1 ] d z +[ 1 ] e z + */ + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + make_attr(loc_id,1,dims,"string",type_id,buf1); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + buf2[i]=buf2[1]=0; + } + /* + buf2[2]= {1,2}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <bitfield> and <bitfield> + position bitfield of </g1> bitfield of </g1> difference + position opaque of </g1> opaque of </g1> difference +------------------------------------------------------------ +[ 0 ] 1 0 1 +[ 1 ] 2 0 2 + */ + + type_id = H5Tcopy(H5T_STD_B8LE); + make_attr(loc_id,1,dims,"bitfield",type_id,buf2); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + + /* + buf2[2]= {1,2}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <opaque> and <opaque> + position opaque of </g1> opaque of </g1> difference + position opaque of </g1> opaque of </g1> difference +------------------------------------------------------------ +[ 0 ] 1 0 1 +[ 1 ] 2 0 2 +*/ + + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + make_attr(loc_id,1,dims,"opaque",type_id,buf2); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf3[i].a=0; buf3[i].b=0; + } + } + + /* + buf3[2]= {{1,2},{3,4}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <compound> and <compound> + position compound of </g1> compound of </g1> difference + ------------------------------------------------------------ + [ 0 ] 1 5 4 + [ 0 ] 2 5 3 + [ 1 ] 3 5 2 + [ 1 ] 4 5 1 + */ + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + make_attr(loc_id,1,dims,"compound",type_id,buf3); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* object references ( H5R_OBJECT */ + if (dset_name) + { + status=H5Rcreate(&buf4[0],fid,dset_name,H5R_OBJECT,-1); + status=H5Rcreate(&buf4[1],fid,dset_name,H5R_OBJECT,-1); + make_attr(loc_id,1,dims,"reference to object",H5T_STD_REF_OBJ,buf4); + } + + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf45[i]=GREEN; + } + } + /* + buf45[2]= {RED,RED}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <enum> and <enum> + position enum of </g1> enum of </g1> difference +------------------------------------------------------------ +[ 0 ] RED GREEN +[ 1 ] RED GREEN + */ + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + make_attr(loc_id,1,dims,"enum",type_id,buf45); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + + /* Allocate and initialize VL dataset to write */ + + buf5[0].len = 1; + buf5[0].p = malloc( 1 * sizeof(int)); + ((int *)buf5[0].p)[0]=1; + buf5[1].len = 2; + buf5[1].p = malloc( 2 * sizeof(int)); + ((int *)buf5[1].p)[0]=2; + ((int *)buf5[1].p)[1]=3; + + if (make_diffs) + { + ((int *)buf5[0].p)[0]=0; + ((int *)buf5[1].p)[0]=0; + ((int *)buf5[1].p)[1]=0; + } + /* + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + position vlen of </g1> vlen of </g1> difference +------------------------------------------------------------ +[ 0 ] 1 0 1 +[ 1 ] 2 0 2 +[ 1 ] 3 0 3 + */ + + space_id = H5Screate_simple(1,dims,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + attr_id = H5Acreate(loc_id,"vlen",type_id,space_id,H5P_DEFAULT); + status = H5Awrite(attr_id,type_id,buf5); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf5); + assert(status>=0); + status = H5Aclose(attr_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + for (j=0; j<3; j++) + { + buf6[i][j]=0; + } + } + /* + buf6[2][3]= {{1,2,3},{4,5,6}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + Attribute: <array> and <array> +position array of </g1> array of </g1> difference +------------------------------------------------------------ +[ 0 ] 1 0 1 +[ 0 ] 2 0 2 +[ 0 ] 3 0 3 +[ 1 ] 4 0 4 +[ 1 ] 5 0 5 +[ 1 ] 6 0 6 + */ + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + make_attr(loc_id,1,dims,"array",type_id,buf6); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER and H5T_FLOAT + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf7[i]=0; + buf8[i]=0; + } + } + /* + buf7[2]= {1,2}; + buf8[2]= {1,2}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> + position integer of </g1> integer of </g1> difference + ------------------------------------------------------------ + [ 0 ] 1 0 1 + [ 1 ] 2 0 2 + position float of </g1> float of </g1> difference + ------------------------------------------------------------ + [ 0 ] 1 0 1 + [ 1 ] 2 0 2 + */ + make_attr(loc_id,1,dims,"integer",H5T_NATIVE_INT,buf7); + make_attr(loc_id,1,dims,"float",H5T_NATIVE_FLOAT,buf8); + + +/*------------------------------------------------------------------------- + * 2D attributes + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + if (make_diffs) + { + memset(buf12, 'z', sizeof buf12); + } + + /* + buf12[6][2]= {"ab","cd","ef","gh","ij","kl"}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Attribute: <string2D> and <string2D> + position string2D of </g1> string2D of </g1> difference + ------------------------------------------------------------ +[ 0 0 ] a z +[ 0 0 ] b z +[ 0 1 ] c z +[ 0 1 ] d z +[ 1 0 ] e z +[ 1 0 ] f z +[ 1 1 ] g z +[ 1 1 ] h z +[ 2 0 ] i z +[ 2 0 ] j z +[ 2 1 ] k z +[ 2 1 ] l z + */ + + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + make_attr(loc_id,2,dims2,"string2D",type_id,buf12); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf22,0,sizeof buf22); + } + + /* + buf22[3][2]= {{1,2},{3,4},{5,6}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Attribute: <bitfield2D> and <bitfield2D> + position bitfield2D of </g1> bitfield2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 1 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 2 0 ] 5 0 5 +[ 2 1 ] 6 0 6 + */ + + + type_id = H5Tcopy(H5T_STD_B8LE); + make_attr(loc_id,2,dims2,"bitfield2D",type_id,buf22); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + + /* + buf22[3][2]= {{1,2},{3,4},{5,6}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Attribute: <opaque2D> and <opaque2D> + position opaque2D of </g1> opaque2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 1 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 2 0 ] 5 0 5 +[ 2 1 ] 6 0 6 + */ + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + make_attr(loc_id,2,dims2,"opaque2D",type_id,buf22); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + if (make_diffs) + { + memset(buf32,0,sizeof buf32); + } + + /* + buf32[6]= {{1,2},{3,4},{5,6},{7,8},{9,10},{11,12}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Attribute: <opaque2D> and <opaque2D> + position opaque2D of </g1> opaque2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 1 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 2 0 ] 5 0 5 +[ 2 1 ] 6 0 6 + */ + + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + make_attr(loc_id,2,dims2,"compound2D",type_id,buf32); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* Create references to dataset */ + if (dset_name) + { + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + status=H5Rcreate(&buf42[i][j],fid,dset_name,H5R_OBJECT,-1); + } + } + make_attr(loc_id,2,dims2,"reference2D",H5T_STD_REF_OBJ,buf42); + } + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + for (i=0; i<3; i++) + for (j=0; j<2; j++) + { + if (make_diffs) buf452[i][j]=GREEN; else buf452[i][j]=RED; + } + +/* +Attribute: <enum2D> and <enum2D> +position enum2D of </g1> enum2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] RED GREEN +[ 0 1 ] RED GREEN +[ 1 0 ] RED GREEN +[ 1 1 ] RED GREEN +[ 2 0 ] RED GREEN +[ 2 1 ] RED GREEN +*/ + + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + make_attr(loc_id,2,dims2,"enum2D",type_id,buf452); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + + /* Allocate and initialize VL dataset to write */ + n=0; + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + int l; + buf52[i][j].p = malloc((i + 1) * sizeof(int)); + buf52[i][j].len = i + 1; + for (l = 0; l < i + 1; l++) + if (make_diffs)((int *)buf52[i][j].p)[l] = 0; + else ((int *)buf52[i][j].p)[l] = n++; + } + } + + /* + position vlen2D of </g1> vlen2D of </g1> difference +------------------------------------------------------------ +[ 0 1 ] 1 0 1 +[ 1 0 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 1 1 ] 5 0 5 +[ 2 0 ] 6 0 6 +[ 2 0 ] 7 0 7 +[ 2 0 ] 8 0 8 +[ 2 1 ] 9 0 9 +[ 2 1 ] 10 0 10 +[ 2 1 ] 11 0 11 +*/ + + space_id = H5Screate_simple(2,dims2,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + attr_id = H5Acreate(loc_id,"vlen2D",type_id,space_id,H5P_DEFAULT); + status = H5Awrite(attr_id,type_id,buf52); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf52); + assert(status>=0); + status = H5Aclose(attr_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf62,0,sizeof buf62); + } + /* + buf62[6][3]= {{1,2,3},{4,5,6},{7,8,9},{10,11,12},{13,14,15},{16,17,18}}; + $h5diff file7.h5 file6.h5 g1 g1 -v + Group: </g1> and </g1> +Attribute: <array2D> and <array2D> +position array2D of </g1> array2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 0 ] 2 0 2 +[ 0 0 ] 3 0 3 +[ 0 1 ] 4 0 4 +[ 0 1 ] 5 0 5 +[ 0 1 ] 6 0 6 +[ 1 0 ] 7 0 7 +[ 1 0 ] 8 0 8 +[ 1 0 ] 9 0 9 +[ 1 1 ] 10 0 10 +[ 1 1 ] 11 0 11 +[ 1 1 ] 12 0 12 +[ 2 0 ] 13 0 13 +[ 2 0 ] 14 0 14 +[ 2 0 ] 15 0 15 +[ 2 1 ] 16 0 16 +[ 2 1 ] 17 0 17 +[ 2 1 ] 18 0 18 + */ + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + make_attr(loc_id,2,dims2,"array2D",type_id,buf62); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER and H5T_FLOAT + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf72,0,sizeof buf72); + memset(buf82,0,sizeof buf82); + } +/* +Attribute: <integer2D> and <integer2D> +position integer2D of </g1> integer2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 1 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 2 0 ] 5 0 5 +[ 2 1 ] 6 0 6 +6 differences found +Attribute: <float2D> and <float2D> +position float2D of </g1> float2D of </g1> difference +------------------------------------------------------------ +[ 0 0 ] 1 0 1 +[ 0 1 ] 2 0 2 +[ 1 0 ] 3 0 3 +[ 1 1 ] 4 0 4 +[ 2 0 ] 5 0 5 +[ 2 1 ] 6 0 6 +*/ + + make_attr(loc_id,2,dims2,"integer2D",H5T_NATIVE_INT,buf72); + make_attr(loc_id,2,dims2,"float2D",H5T_NATIVE_FLOAT,buf82); + + +/*------------------------------------------------------------------------- + * 3D attributes + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf13,'z',sizeof buf13); + } + + /* + buf13[24][2]= {"ab","cd","ef","gh","ij","kl","mn","pq", + "rs","tu","vw","xz","AB","CD","EF","GH", + "IJ","KL","MN","PQ","RS","TU","VW","XZ"}; + +Attribute: <string3D> and <string3D> +position string3D of </g1> string3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] a z +[ 0 0 0 ] b z +[ 0 0 1 ] c z +[ 0 0 1 ] d z +[ 0 1 0 ] e z +[ 0 1 0 ] f z +[ 0 1 1 ] g z +[ 0 1 1 ] h z +[ 0 2 0 ] i z +[ 0 2 0 ] j z +[ 0 2 1 ] k z +[ 0 2 1 ] l z +[ 1 0 0 ] m z +[ 1 0 0 ] n z +[ 1 0 1 ] p z +[ 1 0 1 ] q z +[ 1 1 0 ] r z +[ 1 1 0 ] s z +[ 1 1 1 ] t z +[ 1 1 1 ] u z +[ 1 2 0 ] v z +[ 1 2 0 ] w z +[ 1 2 1 ] x z +[ 2 0 0 ] A z +[ 2 0 0 ] B z +[ 2 0 1 ] C z +[ 2 0 1 ] D z +[ 2 1 0 ] E z +[ 2 1 0 ] F z +[ 2 1 1 ] G z +[ 2 1 1 ] H z +[ 2 2 0 ] I z +[ 2 2 0 ] J z +[ 2 2 1 ] K z +[ 2 2 1 ] L z +[ 3 0 0 ] M z +[ 3 0 0 ] N z +[ 3 0 1 ] P z +[ 3 0 1 ] Q z +[ 3 1 0 ] R z +[ 3 1 0 ] S z +[ 3 1 1 ] T z +[ 3 1 1 ] U z +[ 3 2 0 ] V z +[ 3 2 0 ] W z +[ 3 2 1 ] X z +[ 3 2 1 ] Z z + */ + + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + make_attr(loc_id,3,dims3,"string3D",type_id,buf13); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + n=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) buf23[i][j][k]=0; + else buf23[i][j][k]=n++; + } + } + } + +/* +position bitfield3D of </g1> bitfield3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] 1 0 1 +[ 0 0 1 ] 2 0 2 +[ 0 1 0 ] 3 0 3 +[ 0 1 1 ] 4 0 4 +[ 0 2 0 ] 5 0 5 +[ 0 2 1 ] 6 0 6 +[ 1 0 0 ] 7 0 7 +[ 1 0 1 ] 8 0 8 +[ 1 1 0 ] 9 0 9 +[ 1 1 1 ] 10 0 10 +[ 1 2 0 ] 11 0 11 +[ 1 2 1 ] 12 0 12 +[ 2 0 0 ] 13 0 13 +[ 2 0 1 ] 14 0 14 +[ 2 1 0 ] 15 0 15 +[ 2 1 1 ] 16 0 16 +[ 2 2 0 ] 17 0 17 +[ 2 2 1 ] 18 0 18 +[ 3 0 0 ] 19 0 19 +[ 3 0 1 ] 20 0 20 +[ 3 1 0 ] 21 0 21 +[ 3 1 1 ] 22 0 22 +[ 3 2 0 ] 23 0 23 +[ 3 2 1 ] 24 0 24 +*/ + + type_id = H5Tcopy(H5T_STD_B8LE); + make_attr(loc_id,3,dims3,"bitfield3D",type_id,buf23); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + make_attr(loc_id,3,dims3,"opaque3D",type_id,buf23); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + + n=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) { + buf33[i][j][k].a=0; + buf33[i][j][k].b=0; + } + else { + buf33[i][j][k].a=n++; + buf33[i][j][k].b=n++; + } + } + } + } +/*position compound3D of </g1> compound3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] 1 0 1 +[ 0 0 0 ] 2 0 2 +[ 0 0 1 ] 3 0 3 +[ 0 0 1 ] 4 0 4 +[ 0 1 0 ] 5 0 5 +[ 0 1 0 ] 6 0 6 +[ 0 1 1 ] 7 0 7 +[ 0 1 1 ] 8 0 8 +[ 0 2 0 ] 9 0 9 +[ 0 2 0 ] 10 0 10 +[ 0 2 1 ] 11 0 11 +[ 0 2 1 ] 12 0 12 +[ 1 0 0 ] 13 0 13 +[ 1 0 0 ] 14 0 14 +[ 1 0 1 ] 15 0 15 +[ 1 0 1 ] 16 0 16 +[ 1 1 0 ] 17 0 17 +[ 1 1 0 ] 18 0 18 +[ 1 1 1 ] 19 0 19 +[ 1 1 1 ] 20 0 20 +[ 1 2 0 ] 21 0 21 +[ 1 2 0 ] 22 0 22 +[ 1 2 1 ] 23 0 23 +[ 1 2 1 ] 24 0 24 +[ 2 0 0 ] 25 0 25 +[ 2 0 0 ] 26 0 26 +[ 2 0 1 ] 27 0 27 +[ 2 0 1 ] 28 0 28 +[ 2 1 0 ] 29 0 29 +[ 2 1 0 ] 30 0 30 +[ 2 1 1 ] 31 0 31 +[ 2 1 1 ] 32 0 32 +[ 2 2 0 ] 33 0 33 +[ 2 2 0 ] 34 0 34 +[ 2 2 1 ] 35 0 35 +[ 2 2 1 ] 36 0 36 +[ 3 0 0 ] 37 0 37 +[ 3 0 0 ] 38 0 38 +[ 3 0 1 ] 39 0 39 +[ 3 0 1 ] 40 0 40 +[ 3 1 0 ] 41 0 41 +[ 3 1 0 ] 42 0 42 +[ 3 1 1 ] 43 0 43 +[ 3 1 1 ] 44 0 44 +[ 3 2 0 ] 45 0 45 +[ 3 2 0 ] 46 0 46 +[ 3 2 1 ] 47 0 47 +[ 3 2 1 ] 48 0 48 +*/ + + + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + make_attr(loc_id,3,dims3,"compound3D",type_id,buf33); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* Create references to dataset */ + if (dset_name) + { + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) + status=H5Rcreate(&buf43[i][j][k],fid,dset_name,H5R_OBJECT,-1); + } + } + make_attr(loc_id,3,dims3,"reference3D",H5T_STD_REF_OBJ,buf43); + } + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) buf453[i][j][k]=RED; else buf453[i][j][k]=GREEN; + } + } + } + +/* +position enum3D of </g1> enum3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] GREEN RED +[ 0 0 1 ] GREEN RED +[ 0 1 0 ] GREEN RED +[ 0 1 1 ] GREEN RED +[ 0 2 0 ] GREEN RED +[ 0 2 1 ] GREEN RED +[ 1 0 0 ] GREEN RED +[ 1 0 1 ] GREEN RED +[ 1 1 0 ] GREEN RED +[ 1 1 1 ] GREEN RED +[ 1 2 0 ] GREEN RED +[ 1 2 1 ] GREEN RED +[ 2 0 0 ] GREEN RED +[ 2 0 1 ] GREEN RED +[ 2 1 0 ] GREEN RED +[ 2 1 1 ] GREEN RED +[ 2 2 0 ] GREEN RED +[ 2 2 1 ] GREEN RED +[ 3 0 0 ] GREEN RED +[ 3 0 1 ] GREEN RED +[ 3 1 0 ] GREEN RED +[ 3 1 1 ] GREEN RED +[ 3 2 0 ] GREEN RED +[ 3 2 1 ] GREEN RED +*/ + + + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + make_attr(loc_id,3,dims3,"enum3D",type_id,buf453); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + + /* Allocate and initialize VL dataset to write */ + n=0; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + int l; + buf53[i][j][k].p = malloc((i + 1) * sizeof(int)); + buf53[i][j][k].len = i + 1; + for (l = 0; l < i + 1; l++) + if (make_diffs)((int *)buf53[i][j][k].p)[l] = 0; + else ((int *)buf53[i][j][k].p)[l] = n++; + } + } + } +/* +position vlen3D of </g1> vlen3D of </g1> difference +------------------------------------------------------------ +[ 0 0 1 ] 1 0 1 +[ 0 1 0 ] 2 0 2 +[ 0 1 1 ] 3 0 3 +[ 0 2 0 ] 4 0 4 +[ 0 2 1 ] 5 0 5 +[ 1 0 0 ] 6 0 6 +[ 1 0 0 ] 7 0 7 +[ 1 0 1 ] 8 0 8 +[ 1 0 1 ] 9 0 9 +[ 1 1 0 ] 10 0 10 +etc +*/ + space_id = H5Screate_simple(3,dims3,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + attr_id = H5Acreate(loc_id,"vlen3D",type_id,space_id,H5P_DEFAULT); + status = H5Awrite(attr_id,type_id,buf53); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf53); + assert(status>=0); + status = H5Aclose(attr_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + n=1; + for (i = 0; i < 24; i++) { + for (j = 0; j < (int)dimarray[0]; j++) { + if (make_diffs) buf63[i][j]=0; + else buf63[i][j]=n++; + } + } + /* + position array3D of </g1> array3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] 1 0 1 +[ 0 0 0 ] 2 0 2 +[ 0 0 0 ] 3 0 3 +[ 0 0 1 ] 4 0 4 +[ 0 0 1 ] 5 0 5 +[ 0 0 1 ] 6 0 6 +[ 0 1 0 ] 7 0 7 +etc +*/ + + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + make_attr(loc_id,3,dims3,"array3D",type_id,buf63); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER and H5T_FLOAT + *------------------------------------------------------------------------- + */ + n=1; f=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) { + buf73[i][j][k]=0; + buf83[i][j][k]=0; + } + else { + buf73[i][j][k]=n++; + buf83[i][j][k]=f++; + } + } + } + } + + /* + position integer3D of </g1> integer3D of </g1> difference +------------------------------------------------------------ +[ 0 0 0 ] 1 0 1 +[ 0 0 1 ] 2 0 2 +[ 0 1 0 ] 3 0 3 +[ 0 1 1 ] 4 0 4 +[ 0 2 0 ] 5 0 5 +[ 0 2 1 ] 6 0 6 +[ 1 0 0 ] 7 0 7 +[ 1 0 1 ] 8 0 8 +[ 1 1 0 ] 9 0 9 +[ 1 1 1 ] 10 0 10 +etc +*/ + make_attr(loc_id,3,dims3,"integer3D",H5T_NATIVE_INT,buf73); + make_attr(loc_id,3,dims3,"float3D",H5T_NATIVE_FLOAT,buf83); +} + + + diff --git a/tools/h5repack/testh5repack_dset.c b/tools/h5repack/testh5repack_dset.c new file mode 100644 index 0000000..57a5b27 --- /dev/null +++ b/tools/h5repack/testh5repack_dset.c @@ -0,0 +1,826 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + +static void make_dset_reg_ref(hid_t loc_id); +#ifdef LATER +static void read_dset_reg_ref(hid_t loc_id); +#endif /* LATER */ + + + +/*------------------------------------------------------------------------- + * Function: write_dset_in + * + * Purpose: write datasets in LOC_ID + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: November 12, 2003 + * + *------------------------------------------------------------------------- + */ + + +void write_dset_in(hid_t loc_id, + const char* dset_name, /* for saving reference to dataset*/ + hid_t file_id, + int make_diffs /* flag to modify data buffers */) +{ + /* Compound datatype */ + typedef struct s_t + { + char a; + double b; + } s_t; + + typedef enum + { + RED, + GREEN + } e_t; + + hid_t dset_id; + hid_t space_id; + hid_t type_id; + hid_t plist_id; + herr_t status; + int val, i, j, k, n; + float f; + int fillvalue=2; + + /* create 1D attributes with dimension [2], 2 elements */ + hsize_t dims[1]={2}; + char buf1[2][2]= {"ab","de"}; /* string */ + char buf2[2]= {1,2}; /* bitfield, opaque */ + s_t buf3[2]= {{1,2},{3,4}}; /* compound */ + hobj_ref_t buf4[2]; /* reference */ + e_t buf45[2]= {RED,GREEN}; /* enum */ + hvl_t buf5[2]; /* vlen */ + hsize_t dimarray[1]={3}; /* array dimension */ + int buf6[2][3]= {{1,2,3},{4,5,6}}; /* array */ + int buf7[2]= {1,2}; /* integer */ + float buf8[2]= {1,2}; /* float */ + + /* create 2D attributes with dimension [3][2], 6 elements */ + hsize_t dims2[2]={3,2}; + char buf12[6][2]= {"ab","cd","ef","gh","ij","kl"}; /* string */ + char buf22[3][2]= {{1,2},{3,4},{5,6}}; /* bitfield, opaque */ + s_t buf32[6]= {{1,2},{3,4},{5,6},{7,8},{9,10},{11,12}}; /* compound */ + hobj_ref_t buf42[3][2]; /* reference */ + hvl_t buf52[3][2]; /* vlen */ + int buf62[6][3]= {{1,2,3},{4,5,6},{7,8,9},{10,11,12},{13,14,15},{16,17,18}}; /* array */ + int buf72[3][2]= {{1,2},{3,4},{5,6}}; /* integer */ + float buf82[3][2]= {{1,2},{3,4},{5,6}}; /* float */ + + /* create 3D attributes with dimension [4][3][2], 24 elements */ + hsize_t dims3[3]={4,3,2}; + char buf13[24][2]= {"ab","cd","ef","gh","ij","kl","mn","pq", + "rs","tu","vw","xz","AB","CD","EF","GH", + "IJ","KL","MN","PQ","RS","TU","VW","XZ"}; /* string */ + char buf23[4][3][2]; /* bitfield, opaque */ + s_t buf33[4][3][2]; /* compound */ + hobj_ref_t buf43[4][3][2]; /* reference */ + hvl_t buf53[4][3][2]; /* vlen */ + int buf63[24][3]; /* array */ + int buf73[4][3][2]; /* integer */ + float buf83[4][3][2]; /* float */ + + +/*------------------------------------------------------------------------- + * 1D + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + + + if (make_diffs) + { + for (i=0; i<2; i++) + for (j=0; j<2; j++) + { + buf1[i][j]='z'; + } + } + + + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + write_dset(loc_id,1,dims,"string",type_id,buf1); + status = H5Tclose(type_id); + + + /* create hard link */ + status = H5Glink(loc_id, H5G_LINK_HARD, "string", "string_link"); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + buf2[i]=buf2[1]=0; + } + + type_id = H5Tcopy(H5T_STD_B8LE); + write_dset(loc_id,1,dims,"bitfield",type_id,buf2); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf3[i].a=0; buf3[i].b=0; + } + } + + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + write_dset(loc_id,1,dims,"opaque",type_id,buf2); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + + + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf45[i]=GREEN; + } + } + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + write_dset(loc_id,1,dims,"compound",type_id,buf3); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* object references ( H5R_OBJECT ) */ + if (dset_name) + { + status=H5Rcreate(&buf4[0],file_id,dset_name,H5R_OBJECT,-1); + status=H5Rcreate(&buf4[1],file_id,dset_name,H5R_OBJECT,-1); + write_dset(loc_id,1,dims,"reference to object ",H5T_STD_REF_OBJ,buf4); + } + + /* Dataset region reference ( H5R_DATASET_REGION ) */ + make_dset_reg_ref(loc_id); + + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + write_dset(loc_id,1,dims,"enum",type_id,buf45); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + + /* Allocate and initialize VL dataset to write */ + + buf5[0].len = 1; + buf5[0].p = malloc( 1 * sizeof(int)); + ((int *)buf5[0].p)[0]=1; + buf5[1].len = 2; + buf5[1].p = malloc( 2 * sizeof(int)); + ((int *)buf5[1].p)[0]=2; + ((int *)buf5[1].p)[1]=3; + + if (make_diffs) + { + ((int *)buf5[0].p)[0]=0; + ((int *)buf5[1].p)[0]=0; + ((int *)buf5[1].p)[1]=0; + } + + space_id = H5Screate_simple(1,dims,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + dset_id = H5Dcreate(loc_id,"vlen",type_id,space_id,H5P_DEFAULT); + status = H5Dwrite(dset_id,type_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf5); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf5); + assert(status>=0); + status = H5Dclose(dset_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + for (j=0; j<3; j++) + { + buf6[i][j]=0; + } + } + + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + write_dset(loc_id,1,dims,"array",type_id,buf6); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER and H5T_FLOAT + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + for (i=0; i<2; i++) + { + buf7[i]=0; + buf8[i]=0; + } + } + + write_dset(loc_id,1,dims,"integer",H5T_NATIVE_INT,buf7); + write_dset(loc_id,1,dims,"float",H5T_NATIVE_FLOAT,buf8); + + +/*------------------------------------------------------------------------- + * 2D + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf12, 'z', sizeof buf12); + } + + + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + write_dset(loc_id,2,dims2,"string2D",type_id,buf12); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + + if (make_diffs) + { + memset(buf22,0,sizeof buf22); + } + + type_id = H5Tcopy(H5T_STD_B8LE); + write_dset(loc_id,2,dims2,"bitfield2D",type_id,buf22); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + write_dset(loc_id,2,dims2,"opaque2D",type_id,buf22); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf32,0,sizeof buf32); + } + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + write_dset(loc_id,2,dims2,"compound2D",type_id,buf32); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* Create references to dataset */ + if (dset_name) + { + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + status=H5Rcreate(&buf42[i][j],file_id,dset_name,H5R_OBJECT,-1); + } + } + write_dset(loc_id,2,dims2,"reference to object 2D",H5T_STD_REF_OBJ,buf42); + } + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + write_dset(loc_id,2,dims2,"enum2D",type_id,0); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + +/* Allocate and initialize VL dataset to write */ + n=0; + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + int l; + buf52[i][j].p = malloc((i + 1) * sizeof(int)); + buf52[i][j].len = i + 1; + for (l = 0; l < i + 1; l++) + if (make_diffs)((int *)buf52[i][j].p)[l] = 0; + else ((int *)buf52[i][j].p)[l] = n++; + } + } + + space_id = H5Screate_simple(2,dims2,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + dset_id = H5Dcreate(loc_id,"vlen2D",type_id,space_id,H5P_DEFAULT); + status = H5Dwrite(dset_id,type_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf52); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf52); + assert(status>=0); + status = H5Dclose(dset_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf62,0,sizeof buf62); + } + + + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + write_dset(loc_id,2,dims2,"array2D",type_id,buf62); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER, write a fill value + *------------------------------------------------------------------------- + */ + + + if (make_diffs) + { + memset(buf72,0,sizeof buf72); + memset(buf82,0,sizeof buf82); + } + + + plist_id = H5Pcreate(H5P_DATASET_CREATE); + status = H5Pset_fill_value(plist_id, H5T_NATIVE_INT, &fillvalue); + space_id = H5Screate_simple(2,dims2,NULL); + dset_id = H5Dcreate(loc_id,"integer2D",H5T_NATIVE_INT,space_id,plist_id); + status = H5Dwrite(dset_id,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf72); + status = H5Pclose(plist_id); + status = H5Dclose(dset_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_FLOAT + *------------------------------------------------------------------------- + */ + + write_dset(loc_id,2,dims2,"float2D",H5T_NATIVE_FLOAT,buf82); + + +/*------------------------------------------------------------------------- + * 3D + *------------------------------------------------------------------------- + */ + +/*------------------------------------------------------------------------- + * H5T_STRING + *------------------------------------------------------------------------- + */ + + if (make_diffs) + { + memset(buf13,'z',sizeof buf13); + } + + type_id = H5Tcopy(H5T_C_S1); + status = H5Tset_size(type_id, 2); + write_dset(loc_id,3,dims3,"string3D",type_id,buf13); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_BITFIELD + *------------------------------------------------------------------------- + */ + + + n=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) buf23[i][j][k]=0; + else buf23[i][j][k]=n++; + } + } + } + + + type_id = H5Tcopy(H5T_STD_B8LE); + write_dset(loc_id,3,dims3,"bitfield3D",type_id,buf23); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_OPAQUE + *------------------------------------------------------------------------- + */ + type_id = H5Tcreate(H5T_OPAQUE, 1); + status = H5Tset_tag(type_id, "1-byte opaque type"); /* must set this */ + write_dset(loc_id,3,dims3,"opaque3D",type_id,buf23); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_COMPOUND + *------------------------------------------------------------------------- + */ + + n=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) { + buf33[i][j][k].a=0; + buf33[i][j][k].b=0; + } + else { + buf33[i][j][k].a=n++; + buf33[i][j][k].b=n++; + } + } + } + } + + + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_CHAR); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_DOUBLE); + write_dset(loc_id,3,dims3,"compound3D",type_id,buf33); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_REFERENCE + *------------------------------------------------------------------------- + */ + /* Create references to dataset */ + if (dset_name) + { + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) + status=H5Rcreate(&buf43[i][j][k],file_id,dset_name,H5R_OBJECT,-1); + } + } + write_dset(loc_id,3,dims3,"reference to object 3D",H5T_STD_REF_OBJ,buf43); + } + +/*------------------------------------------------------------------------- + * H5T_ENUM + *------------------------------------------------------------------------- + */ + + type_id = H5Tcreate(H5T_ENUM, sizeof(e_t)); + H5Tenum_insert(type_id, "RED", (val = 0, &val)); + H5Tenum_insert(type_id, "GREEN", (val = 1, &val)); + write_dset(loc_id,3,dims3,"enum3D",type_id,0); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_VLEN + *------------------------------------------------------------------------- + */ + + /* Allocate and initialize VL dataset to write */ + n=0; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + int l; + buf53[i][j][k].p = malloc((i + 1) * sizeof(int)); + buf53[i][j][k].len = i + 1; + for (l = 0; l < i + 1; l++) + if (make_diffs)((int *)buf53[i][j][k].p)[l] = 0; + else ((int *)buf53[i][j][k].p)[l] = n++; + } + } + } + + space_id = H5Screate_simple(3,dims3,NULL); + type_id = H5Tvlen_create(H5T_NATIVE_INT); + dset_id = H5Dcreate(loc_id,"vlen3D",type_id,space_id,H5P_DEFAULT); + status = H5Dwrite(dset_id,type_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf53); + assert(status>=0); + status = H5Dvlen_reclaim(type_id,space_id,H5P_DEFAULT,buf53); + assert(status>=0); + status = H5Dclose(dset_id); + status = H5Tclose(type_id); + status = H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5T_ARRAY + *------------------------------------------------------------------------- + */ + + + n=1; + for (i = 0; i < 24; i++) { + for (j = 0; j < (int)dimarray[0]; j++) { + if (make_diffs) buf63[i][j]=0; + else buf63[i][j]=n++; + } + } + + type_id = H5Tarray_create(H5T_NATIVE_INT,1,dimarray,NULL); + write_dset(loc_id,3,dims3,"array3D",type_id,buf63); + status = H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER and H5T_FLOAT + *------------------------------------------------------------------------- + */ + n=1; f=1; + for (i = 0; i < 4; i++) { + for (j = 0; j < 3; j++) { + for (k = 0; k < 2; k++) { + if (make_diffs) { + buf73[i][j][k]=0; + buf83[i][j][k]=0; + } + else { + buf73[i][j][k]=n++; + buf83[i][j][k]=f++; + } + } + } + } + write_dset(loc_id,3,dims3,"integer3D",H5T_NATIVE_INT,buf73); + write_dset(loc_id,3,dims3,"float3D",H5T_NATIVE_FLOAT,buf83); +} + + + +/*------------------------------------------------------------------------- + * Function: make_dset_reg_ref + * + * Purpose: write dataset region references + * + *------------------------------------------------------------------------- + */ + +#define SPACE1_RANK 1 +#define SPACE1_DIM1 4 +#define SPACE2_RANK 2 +#define SPACE2_DIM1 10 +#define SPACE2_DIM2 10 +#define NPOINTS 10 + +static void make_dset_reg_ref(hid_t loc_id) +{ + hid_t dset1; /* Dataset ID */ + hid_t dset2; /* Dereferenced dataset ID */ + hid_t sid1; /* Dataspace ID #1 */ + hid_t sid2; /* Dataspace ID #2 */ + hsize_t dims1[] = {SPACE1_DIM1}; + hsize_t dims2[] = {SPACE2_DIM1, SPACE2_DIM2}; + hssize_t start[SPACE2_RANK]; /* Starting location of hyperslab */ + hsize_t stride[SPACE2_RANK]; /* Stride of hyperslab */ + hsize_t count[SPACE2_RANK]; /* Element count of hyperslab */ + hsize_t block[SPACE2_RANK]; /* Block size of hyperslab */ + hssize_t coord1[NPOINTS][SPACE2_RANK]; /* Coordinates for point selection */ + hdset_reg_ref_t *wbuf; /* buffer to write to disk */ + int *dwbuf; /* Buffer for writing numeric data to disk */ + int i; /* counting variables */ + herr_t ret; /* Generic return value */ + + /* Allocate write & read buffers */ + wbuf=calloc(sizeof(hdset_reg_ref_t), SPACE1_DIM1); + dwbuf=malloc(sizeof(int)*SPACE2_DIM1*SPACE2_DIM2); + + /* Create dataspace for datasets */ + sid2 = H5Screate_simple(SPACE2_RANK, dims2, NULL); + + /* Create a dataset */ + dset2=H5Dcreate(loc_id,"dset referenced region",H5T_STD_U8LE,sid2,H5P_DEFAULT); + + for(i=0; i<SPACE2_DIM1*SPACE2_DIM2; i++) + dwbuf[i]=i*3; + + /* Write selection to disk */ + ret=H5Dwrite(dset2,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,dwbuf); + + /* Close Dataset */ + ret = H5Dclose(dset2); + + /* Create dataspace for the reference dataset */ + sid1 = H5Screate_simple(SPACE1_RANK, dims1, NULL); + + /* Create a dataset */ + dset1=H5Dcreate(loc_id,"region reference",H5T_STD_REF_DSETREG,sid1,H5P_DEFAULT); + + /* Create references */ + + /* Select 6x6 hyperslab for first reference */ + start[0]=2; start[1]=2; + stride[0]=1; stride[1]=1; + count[0]=6; count[1]=6; + block[0]=1; block[1]=1; + ret = H5Sselect_hyperslab(sid2,H5S_SELECT_SET,start,stride,count,block); + + /* Store first dataset region */ + ret = H5Rcreate(&wbuf[0],loc_id,"dset referenced region",H5R_DATASET_REGION,sid2); + + /* Select sequence of ten points for second reference */ + coord1[0][0]=6; coord1[0][1]=9; + coord1[1][0]=2; coord1[1][1]=2; + coord1[2][0]=8; coord1[2][1]=4; + coord1[3][0]=1; coord1[3][1]=6; + coord1[4][0]=2; coord1[4][1]=8; + coord1[5][0]=3; coord1[5][1]=2; + coord1[6][0]=0; coord1[6][1]=4; + coord1[7][0]=9; coord1[7][1]=0; + coord1[8][0]=7; coord1[8][1]=1; + coord1[9][0]=3; coord1[9][1]=3; + ret = H5Sselect_elements(sid2,H5S_SELECT_SET,NPOINTS,(const hssize_t **)coord1); + + /* Store second dataset region */ + ret = H5Rcreate(&wbuf[1],loc_id,"dset referenced region",H5R_DATASET_REGION,sid2); + + /* Write selection to disk */ + ret=H5Dwrite(dset1,H5T_STD_REF_DSETREG,H5S_ALL,H5S_ALL,H5P_DEFAULT,wbuf); + + /* Close all objects */ + ret = H5Sclose(sid1); + ret = H5Dclose(dset1); + ret = H5Sclose(sid2); + + free(wbuf); + free(dwbuf); +} + +/*------------------------------------------------------------------------- + * Function: read_dset_reg_ref + * + * Purpose: read dataset region references + * + *------------------------------------------------------------------------- + */ + +#ifdef LATER +static void read_dset_reg_ref(hid_t loc_id) +{ + hid_t dset1; /* Dataset ID */ + hid_t dset2; /* Dereferenced dataset ID */ + hid_t sid1; /* Dataspace ID #1 */ + hid_t sid2; /* Dataspace ID #2 */ + hsize_t *coords; /* Coordinate buffer */ + hssize_t low[SPACE2_RANK]; /* Selection bounds */ + hssize_t high[SPACE2_RANK]; /* Selection bounds */ + hdset_reg_ref_t *rbuf; /* buffer to read from disk */ + int *drbuf; /* Buffer for reading numeric data from disk */ + int i, j; /* counting variables */ + herr_t ret; /* Generic return value */ + + /* Allocate write & read buffers */ + rbuf=malloc(sizeof(hdset_reg_ref_t)*SPACE1_DIM1); + drbuf=calloc(sizeof(int),SPACE2_DIM1*SPACE2_DIM2); + + /* Open the dataset */ + dset1=H5Dopen(loc_id,"region reference"); + + /* Read selection from disk */ + ret=H5Dread(dset1,H5T_STD_REF_DSETREG,H5S_ALL,H5S_ALL,H5P_DEFAULT,rbuf); + + /* Try to open objects */ + dset2 = H5Rdereference(dset1,H5R_DATASET_REGION,&rbuf[0]); + + /* Check information in referenced dataset */ + sid1 = H5Dget_space(dset2); + + ret=(herr_t)H5Sget_simple_extent_npoints(sid1); + printf(" Number of elements in the dataset is : %d\n",ret); + + /* Read from disk */ + ret=H5Dread(dset2,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,drbuf); + + for(i=0; i<SPACE2_DIM1; i++) { + for (j=0; j<SPACE2_DIM2; j++) printf (" %d ", drbuf[i*SPACE2_DIM2+j]); + printf("\n"); } + + /* Get the hyperslab selection */ + sid2=H5Rget_region(dset1,H5R_DATASET_REGION,&rbuf[0]); + + /* Verify correct hyperslab selected */ + ret = (herr_t)H5Sget_select_npoints(sid2); + printf(" Number of elements in the hyperslab is : %d \n", ret); + ret = (herr_t)H5Sget_select_hyper_nblocks(sid2); + /* allocate space for the hyperslab blocks */ + coords=malloc((size_t)ret*SPACE2_RANK*sizeof(hsize_t)*2); + ret = H5Sget_select_hyper_blocklist(sid2,0,ret,coords); + printf(" Hyperslab coordinates are : \n"); + printf (" ( %lu , %lu ) ( %lu , %lu ) \n", + (unsigned long)coords[0], + (unsigned long)coords[1], + (unsigned long)coords[2], + (unsigned long)coords[3]); + free(coords); + ret = H5Sget_select_bounds(sid2,low,high); + + /* Close region space */ + ret = H5Sclose(sid2); + + /* Get the element selection */ + sid2=H5Rget_region(dset1,H5R_DATASET_REGION,&rbuf[1]); + + /* Verify correct elements selected */ + ret = (herr_t)H5Sget_select_elem_npoints(sid2); + printf(" Number of selected elements is : %d\n", ret); + + /* Allocate space for the element points */ + coords= malloc(ret*SPACE2_RANK*sizeof(hsize_t)); + ret = H5Sget_select_elem_pointlist(sid2,0,ret,coords); + printf(" Coordinates of selected elements are : \n"); + for (i=0; i<2*NPOINTS; i=i+2) + printf(" ( %lu , %lu ) \n", + (unsigned long)coords[i], + (unsigned long)coords[i+1]); + + free(coords); + ret = H5Sget_select_bounds(sid2,low,high); + + /* Close region space */ + ret = H5Sclose(sid2); + + /* Close first space */ + ret = H5Sclose(sid1); + + /* Close dereferenced Dataset */ + ret = H5Dclose(dset2); + + /* Close Dataset */ + ret = H5Dclose(dset1); + + /* Free memory buffers */ + free(rbuf); + free(drbuf); +} +#endif /* LATER */ diff --git a/tools/h5repack/testh5repack_filters.c b/tools/h5repack/testh5repack_filters.c new file mode 100644 index 0000000..b285b28 --- /dev/null +++ b/tools/h5repack/testh5repack_filters.c @@ -0,0 +1,166 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + +#define DIM1 40 +#define DIM2 20 +#define CDIM1 DIM1/2 +#define CDIM2 DIM2/2 +#define RANK 2 + +/*------------------------------------------------------------------------- + * Function: make_filters + * + * Purpose: make several datasets with filters in location LOC_ID + * + *------------------------------------------------------------------------- + */ +int make_filters(hid_t loc_id) +{ + hid_t dcpl; /* dataset creation property list */ + hid_t sid; /* dataspace ID */ + unsigned szip_options_mask=H5_SZIP_ALLOW_K13_OPTION_MASK|H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block=8; + hsize_t dims[RANK]={DIM1,DIM2}; + hsize_t chunk_dims[RANK]={CDIM1,CDIM2}; + int buf[DIM1][DIM2]; + char name[5]; + int i, j, n; + + for (i=n=0; i<DIM1; i++){ + for (j=0; j<DIM2; j++){ + buf[i][j]=n++; + } + } + +/*------------------------------------------------------------------------- + * make several dataset with no filters + *------------------------------------------------------------------------- + */ + for (i=0; i<4; i++) + { + sprintf(name,"dset%d",i+1); + if (write_dset(loc_id,RANK,dims,name,H5T_NATIVE_INT,buf)<0) + return -1; + } + +/*------------------------------------------------------------------------- + * make several dataset with filters + *------------------------------------------------------------------------- + */ + /* create a space */ + if((sid = H5Screate_simple(RANK, dims, NULL))<0) + return -1; + /* create a dataset creation property list; the same DCPL is used for all dsets */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + /* set up chunk */ + if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0) + goto out; + +/*------------------------------------------------------------------------- + * SZIP + *------------------------------------------------------------------------- + */ + /* set szip data */ + if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0) + goto out; + if (make_dset(loc_id,"dset_szip",sid,dcpl,buf)<0) + goto out; + +/*------------------------------------------------------------------------- + * GZIP + *------------------------------------------------------------------------- + */ + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set deflate data */ + if(H5Pset_deflate(dcpl, 9)<0) + goto out; + if (make_dset(loc_id,"dset_gzip",sid,dcpl,buf)<0) + goto out; + + +/*------------------------------------------------------------------------- + * shuffle + *------------------------------------------------------------------------- + */ + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set the shuffle filter */ + if (H5Pset_shuffle(dcpl)<0) + goto out; + if (make_dset(loc_id,"dset_shuffle",sid,dcpl,buf)<0) + goto out; + +/*------------------------------------------------------------------------- + * checksum + *------------------------------------------------------------------------- + */ +#if defined (H5_HAVE_FILTER_FLETCHER32) + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set the checksum filter */ + if (H5Pset_fletcher32(dcpl)<0) + goto out; + if (make_dset(loc_id,"dset_fletcher32",sid,dcpl,buf)<0) + goto out; +#endif + +/*------------------------------------------------------------------------- + * shuffle + SZIP + *------------------------------------------------------------------------- + */ + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set the shuffle filter */ + if (H5Pset_shuffle(dcpl)<0) + goto out; + /* set szip data */ + if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0) + goto out; + if (make_dset(loc_id,"dset_all",sid,dcpl,buf)<0) + goto out; + + +/*------------------------------------------------------------------------- + * close space and dcpl + *------------------------------------------------------------------------- + */ + if(H5Sclose(sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + + return 0; + +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + } H5E_END_TRY; + return -1; +} + + + + + diff --git a/tools/h5repack/testh5repack_layout.c b/tools/h5repack/testh5repack_layout.c new file mode 100644 index 0000000..96e0ba6 --- /dev/null +++ b/tools/h5repack/testh5repack_layout.c @@ -0,0 +1,108 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + +#define DIM1 40 +#define DIM2 20 +#define CDIM1 20 +#define CDIM2 10 + +#define RANK 2 + + +/*------------------------------------------------------------------------- + * Function: make_layout + * + * Purpose: make several datasets with several layouts in location LOC_ID + * + *------------------------------------------------------------------------- + */ +int make_layout(hid_t loc_id) +{ + hid_t dcpl; /* dataset creation property list */ + hid_t sid; /* dataspace ID */ + hsize_t dims[RANK]={DIM1,DIM2}; + hsize_t chunk_dims[RANK]={CDIM1,CDIM2}; + int buf[DIM1][DIM2]; + int i, j, n; + + for (i=n=0; i<DIM1; i++){ + for (j=0; j<DIM2; j++){ + buf[i][j]=n++; + } + } +/*------------------------------------------------------------------------- + * make several dataset with several layout options + *------------------------------------------------------------------------- + */ + /* create a space */ + if((sid = H5Screate_simple(RANK, dims, NULL))<0) + return -1; + /* create a dataset creation property list; the same DCPL is used for all dsets */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + +/*------------------------------------------------------------------------- + * H5D_COMPACT + *------------------------------------------------------------------------- + */ + if(H5Pset_layout (dcpl,H5D_COMPACT)<0) + goto out; + if (make_dset(loc_id,"dset_compact",sid,dcpl,buf)<0) + goto out; + +/*------------------------------------------------------------------------- + * H5D_COMPACT + *------------------------------------------------------------------------- + */ + if(H5Pset_layout (dcpl,H5D_CONTIGUOUS)<0) + goto out; + if (make_dset(loc_id,"dset_contiguous",sid,dcpl,buf)<0) + goto out; + +/*------------------------------------------------------------------------- + * H5D_CHUNKED + *------------------------------------------------------------------------- + */ + if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0) + goto out; + if (make_dset(loc_id,"dset_chunk",sid,dcpl,buf)<0) + goto out; + +/*------------------------------------------------------------------------- + * close space and dcpl + *------------------------------------------------------------------------- + */ + if(H5Sclose(sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + + return 0; + +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + } H5E_END_TRY; + return -1; +} + + + + + diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c new file mode 100644 index 0000000..ddecd98 --- /dev/null +++ b/tools/h5repack/testh5repack_main.c @@ -0,0 +1,933 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" +#include "h5diff.h" + +#if 0 +#define PACK_DEBUG +#endif + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Executes h5repack tests + * + * Return: Success: zero + * Failure: non-zero + * + * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu> + * January, 6, 2004 + * + *------------------------------------------------------------------------- + */ + + +int main (void) +{ + pack_opt_t pack_options; + diff_opt_t diff_options; + memset(&diff_options, 0, sizeof (diff_opt_t)); + memset(&pack_options, 0, sizeof (pack_opt_t)); + + /* run tests */ + puts("Testing h5repack:"); + + /* make the test files */ + if (make_testfiles()<0) + goto error; + +/*------------------------------------------------------------------------- + * Purpose: + * + * 1) make a copy with no filters + * 2) use the h5diff utility to compare the input and output file; + * it returns RET==0 if the objects have the same data + *------------------------------------------------------------------------- + */ + + TESTING(" copy of datasets"); + +/*------------------------------------------------------------------------- + * file with all kinds of dataset datatypes + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME1,FNAME1OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME1,FNAME1OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME1OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_cmpdcpl(FNAME1,FNAME1OUT)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" copy of attributes"); +/*------------------------------------------------------------------------- + * file with attributes + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME2,FNAME2OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME2,FNAME2OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME2OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_cmpdcpl(FNAME2,FNAME2OUT)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + TESTING(" copy of hardlinks"); + +/*------------------------------------------------------------------------- + * file with hardlinks + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME3,FNAME3OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME3,FNAME3OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME3OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_cmpdcpl(FNAME3,FNAME3OUT)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + +/*------------------------------------------------------------------------- + * file with filters. we cannot compare with cmpdcpl, because the current + * configuration might not have saved datasets with deflate and SZIP filters + *------------------------------------------------------------------------- + */ + PASSED(); + TESTING(" copy of datasets with all filters"); + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" copy of allocation early file"); + +/*------------------------------------------------------------------------- + * alloc early test + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME5,FNAME5OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME5,FNAME5OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME5OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" removing all filters"); + +/*------------------------------------------------------------------------- + * test the NONE global option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + + TESTING(" removing deflate filter"); + +#ifdef H5_HAVE_FILTER_DEFLATE + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_gzip:NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" removing szip filter"); + +#ifdef H5_HAVE_FILTER_SZIP + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_szip:NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" removing shuffle filter"); + +#ifdef H5_HAVE_FILTER_SHUFFLE + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_shuffle:NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" removing fletcher filter"); + +#ifdef H5_HAVE_FILTER_FLETCHER32 + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_fletcher32:NONE",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + + + TESTING(" adding deflate filter"); + +#ifdef H5_HAVE_FILTER_DEFLATE + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset1:GZIP=9",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); +#else + SKIPPED(); +#endif + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + TESTING(" adding deflate filter to all"); + +#ifdef H5_HAVE_FILTER_DEFLATE + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("GZIP=9",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + +/*------------------------------------------------------------------------- + * deflate + *------------------------------------------------------------------------- + */ + + TESTING(" adding szip filter"); + +#ifdef H5_HAVE_FILTER_SZIP + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset2:SZIP=8",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("dset2:CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ +TESTING(" adding szip filter to all"); + +#ifdef H5_HAVE_FILTER_SZIP + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("SZIP=8",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + + TESTING(" addding shuffle filter"); + +#ifdef H5_HAVE_FILTER_SHUFFLE + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset1:SHUF",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + +TESTING(" addding shuffle filter to all"); + +#ifdef H5_HAVE_FILTER_SHUFFLE + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("SHUF",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + + + TESTING(" adding checksum filter"); + +#ifdef H5_HAVE_FILTER_FLETCHER32 + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset1:FLET",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + + TESTING(" adding checksum filter to all"); + +#ifdef H5_HAVE_FILTER_FLETCHER32 + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("FLET",&pack_options)<0) + TEST_ERROR; + if (h5repack_addlayout("CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + + TESTING(" filter queue fletcher, shuffle, deflate, szip"); + +/*------------------------------------------------------------------------- + * add some filters + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options)<0) + TEST_ERROR; +#if defined (H5_HAVE_FILTER_FLETCHER32) + if (h5repack_addfilter("dset1:FLET",&pack_options)<0) + TEST_ERROR; +#endif + if (h5repack_addfilter("dset1:SHUF",&pack_options)<0) + TEST_ERROR; + if (h5repack_addfilter("dset1:SZIP=8",&pack_options)<0) + TEST_ERROR; + if (h5repack_addfilter("dset1:GZIP=1",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" filter conversion from deflate to szip"); +/*------------------------------------------------------------------------- + * filter conversion from deflate to szip + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_DEFLATE + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_gzip:SZIP=8",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" filter conversion from szip to deflate"); +/*------------------------------------------------------------------------- + * filter conversion from szip to deflate + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_SZIP + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" filter conversion from szip and shuffle to deflate"); +/*------------------------------------------------------------------------- + * filter conversion from szip to deflate + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_SZIP + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset_all:GZIP=1",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" adding layout chunked"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + TESTING(" adding layout chunked to all"); + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("CHUNK=20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" adding layout contiguous"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" adding layout contiguous to all"); + + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + TESTING(" adding layout compact"); + +/*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset1:COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" adding layout compact to all"); + +/*------------------------------------------------------------------------- + * test all objects option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + + PASSED(); + + + TESTING(" layout compact to contiguous conversion"); + +/*------------------------------------------------------------------------- + * layout compact to contiguous conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_compact:CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout compact to chunk conversion"); + +/*------------------------------------------------------------------------- + * layout compact to chunk conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_compact:CHUNK=2x5",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout compact to compact conversion"); + +/*------------------------------------------------------------------------- + * layout compact to compact conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_compact:COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout contiguous to compact conversion"); +/*------------------------------------------------------------------------- + * layout contiguous to compact conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_contiguous:COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout contiguous to chunk conversion"); +/*------------------------------------------------------------------------- + * layout contiguous to chunk conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_contiguous:CHUNK=3x6",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout contiguous to contiguous conversion"); + +/*------------------------------------------------------------------------- + * layout contiguous to contiguous conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_contiguous:CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout chunked to compact conversion"); +/*------------------------------------------------------------------------- + * layout chunked to compact conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_chunk:COMPA",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout chunked to contiguous conversion"); + +/*------------------------------------------------------------------------- + * layout chunked to contiguous conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_chunk:CONTI",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + TESTING(" layout chunked to chunk conversion"); +/*------------------------------------------------------------------------- + * layout chunked to chunked conversion + *------------------------------------------------------------------------- + */ + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addlayout("dset_chunk:CHUNK=18x13",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + + + +/*------------------------------------------------------------------------- + * end + *------------------------------------------------------------------------- + */ + + puts("All h5repack tests passed."); + + return 0; + +error: + puts("***** H5REPACK TESTS FAILED *****"); + return 1; + +} + + + diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c new file mode 100644 index 0000000..c69abd9 --- /dev/null +++ b/tools/h5repack/testh5repack_make.c @@ -0,0 +1,400 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + +int make_all_objects(hid_t loc_id); +int make_attributes(hid_t loc_id); +int make_special_objects(hid_t loc_id); +int make_early(void); + + + +/*------------------------------------------------------------------------- + * Function: make_testfiles + * + * Purpose: make a test file with all types of HDF5 objects, + * datatypes and filters + * + *------------------------------------------------------------------------- + */ +int make_testfiles(void) +{ + hid_t loc_id; /* file ID */ + + TESTING(" generating datasets"); + +/*------------------------------------------------------------------------- + * create a file for general copy test + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME1,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_all_objects(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; + +/*------------------------------------------------------------------------- + * create a file for attributes copy test + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME2,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_attributes(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; +/*------------------------------------------------------------------------- + * create a file for special items test + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME3,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_special_objects(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; +/*------------------------------------------------------------------------- + * create a file for the filters and layouts test + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME4,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_filters(loc_id)<0) + goto out; + if (make_layout(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; + +/*------------------------------------------------------------------------- + * create a file for the H5D_ALLOC_TIME_EARLY test + *------------------------------------------------------------------------- + */ + if (make_early()<0) + goto out; + + + PASSED(); + return 0; + +out: + H5Fclose(loc_id); + return -1; +} + + + +/*------------------------------------------------------------------------- + * Function: make_all_objects + * + * Purpose: make a test file with all types of HDF5 objects, datatypes + * + *------------------------------------------------------------------------- + */ +int make_all_objects(hid_t loc_id) +{ + hid_t dset_id; + hid_t group_id; + hid_t type_id; + hid_t root_id; + hid_t space_id; + hsize_t dims[1]={2}; + /* Compound datatype */ + typedef struct s_t + { + int a; + float b; + } s_t; + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + space_id = H5Screate_simple(1,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset_referenced",H5T_NATIVE_INT,space_id,H5P_DEFAULT); + H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + group_id = H5Gcreate(loc_id,"g1",0); + root_id = H5Gopen(loc_id, "/"); + +/*------------------------------------------------------------------------- + * H5G_TYPE + *------------------------------------------------------------------------- + */ + + /* Create a memory compound datatype */ + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_INT); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_FLOAT); + /* Commit compound datatype and close it */ + H5Tcommit(loc_id, "type", type_id); + H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5G_LINK + *------------------------------------------------------------------------- + */ + + H5Glink(loc_id, H5G_LINK_SOFT, "dset", "link"); + +/*------------------------------------------------------------------------- + * write a series of datasetes on the group, and root group + *------------------------------------------------------------------------- + */ + + write_dset_in(root_id,"dset_referenced",loc_id,0); + write_dset_in(group_id,"dset_referenced",loc_id,0); + + + /* Close */ + H5Dclose(dset_id); + H5Gclose(group_id); + H5Gclose(root_id); + + return 0; + +} + + +/*------------------------------------------------------------------------- + * Function: make_attributes + * + * Purpose: make a test file with all types of attributes + * + *------------------------------------------------------------------------- + */ +int make_attributes(hid_t loc_id) +{ + hid_t dset_id; + hid_t group_id; + hid_t root_id; + hid_t space_id; + hsize_t dims[1]={2}; + + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + + space_id = H5Screate_simple(1,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset",H5T_NATIVE_INT,space_id,H5P_DEFAULT); + H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + group_id = H5Gcreate(loc_id,"g1",0); + root_id = H5Gopen(loc_id, "/"); + +/*------------------------------------------------------------------------- + * write a series of attributes on the dataset, group, and root group + *------------------------------------------------------------------------- + */ + + write_attr_in(dset_id,"dset",loc_id,0); + write_attr_in(group_id,"dset",loc_id,0); + write_attr_in(root_id,"dset",loc_id,0); + + /* Close */ + H5Dclose(dset_id); + H5Gclose(group_id); + H5Gclose(root_id); + + return 0; + +} + + + +/*------------------------------------------------------------------------- + * Function: make_special_objects + * + * Purpose: make a test file with non common items + * + *------------------------------------------------------------------------- + */ +int make_special_objects(hid_t loc_id) +{ + hid_t group1_id; + hid_t group2_id; + hid_t group3_id; + hsize_t dims[2]={3,2}; + int buf[3][2]= {{1,1},{1,2},{2,2}}; + hid_t dset_id; + hid_t space_id; + hid_t plist_id; + int fillvalue=2; + +/*------------------------------------------------------------------------- + * create a dataset and some hard links to it + *------------------------------------------------------------------------- + */ + + if (write_dset(loc_id,2,dims,"dset",H5T_NATIVE_INT,buf)<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link1 to dset")<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link2 to dset")<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link3 to dset")<0) + return -1; + + +/*------------------------------------------------------------------------- + * create a group and some hard links to it + *------------------------------------------------------------------------- + */ + + if ((group1_id = H5Gcreate(loc_id,"g1",0))<0) + return -1; + if ((group2_id = H5Gcreate(group1_id,"g2",0))<0) + return -1; + if ((group3_id = H5Gcreate(group2_id,"g3",0))<0) + return -1; + + if (H5Glink2(loc_id, "g1", H5G_LINK_HARD, group2_id, "link1 to g1")<0) + return -1; + if (H5Glink2(group1_id, "g2", H5G_LINK_HARD, group3_id, "link1 to g2")<0) + return -1; + + H5Gclose(group1_id); + H5Gclose(group2_id); + H5Gclose(group3_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER, write a fill value + *------------------------------------------------------------------------- + */ + + plist_id = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_fill_value(plist_id, H5T_NATIVE_INT, &fillvalue); + space_id = H5Screate_simple(2,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,space_id,plist_id); + H5Dwrite(dset_id,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf); + H5Pclose(plist_id); + H5Dclose(dset_id); + H5Sclose(space_id); + + return 0; + +} + + +/*------------------------------------------------------------------------- + * Function: make_early + * + * Purpose: create a file for the H5D_ALLOC_TIME_EARLY test + * + *------------------------------------------------------------------------- + */ +int make_early(void) +{ + hsize_t dims[1] ={3000}; + hsize_t cdims[1]={30}; + hid_t fid=-1; + hid_t dset_id=-1; + hid_t sid=-1; + hid_t tid=-1; + hid_t dcpl=-1; + int i; + char name[10]; + int iter=100; + + if ((fid = H5Fcreate(FNAME5,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (H5Fclose(fid)<0) + goto out; + + if ((sid = H5Screate_simple(1, dims, NULL))<0) + goto out; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + if (H5Pset_chunk(dcpl,1,cdims)<0) + goto out; + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY)<0) + goto out; + + for (i=0; i<iter; i++) + { + if ((fid = H5Fopen(FNAME5,H5F_ACC_RDWR,H5P_DEFAULT))<0) + goto out; + if ((dset_id = H5Dcreate(fid,"early",H5T_NATIVE_DOUBLE,sid,dcpl))<0) + goto out; + if ((tid = H5Tcopy(H5T_NATIVE_DOUBLE))<0) + goto out; + sprintf(name,"%d", i); + if ((H5Tcommit(fid,name,tid))<0) + goto out; + if (H5Tclose(tid)<0) + goto out; + if (H5Dclose(dset_id)<0) + goto out; + if (H5Gunlink(fid,"early")<0) + goto out; + if (H5Fclose(fid)<0) + goto out; + } + +/*------------------------------------------------------------------------- + * do the same without close/opening the file and creating the dataset + *------------------------------------------------------------------------- + */ + + if ((fid = H5Fcreate(FNAME6,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + + for (i=0; i<iter; i++) + { + if ((tid = H5Tcopy(H5T_NATIVE_DOUBLE))<0) + goto out; + sprintf(name,"%d", i); + if ((H5Tcommit(fid,name,tid))<0) + goto out; + if (H5Tclose(tid)<0) + goto out; + } + + if (H5Sclose(sid)<0) + goto out; + if (H5Pclose(dcpl)<0) + goto out; + if (H5Fclose(fid)<0) + goto out; + + + return 0; + +out: + H5E_BEGIN_TRY { + H5Tclose(tid); + H5Pclose(dcpl); + H5Sclose(sid); + H5Dclose(dset_id); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} + + diff --git a/tools/h5repack/testh5repack_util.c b/tools/h5repack/testh5repack_util.c new file mode 100644 index 0000000..f74b414 --- /dev/null +++ b/tools/h5repack/testh5repack_util.c @@ -0,0 +1,153 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5repack.h" + +/*------------------------------------------------------------------------- + * Function: make_dset + * + * Purpose: utility function to create and write a dataset in LOC_ID + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: November 12, 2003 + * + *------------------------------------------------------------------------- + */ +int make_dset(hid_t loc_id, + const char *name, + hid_t sid, + hid_t dcpl, + void *buf) +{ + hid_t dsid; + + /* create the dataset */ + if((dsid = H5Dcreate (loc_id,name,H5T_NATIVE_INT,sid,dcpl))<0) + return -1; + + /* write */ + if(H5Dwrite(dsid,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto out; + + /* close */ + if(H5Dclose(dsid)<0) + return -1; + + return 0; + out: + H5E_BEGIN_TRY { + H5Dclose(dsid); + } H5E_END_TRY; + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: write_dset + * + * Purpose: utility function to create and write a dataset in LOC_ID + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: November 12, 2003 + * + *------------------------------------------------------------------------- + */ + +int write_dset( hid_t loc_id, + int rank, + hsize_t *dims, + const char *dset_name, + hid_t type_id, + void *buf ) +{ + hid_t dset_id; + hid_t space_id; + + /* Create a buf space */ + if ((space_id = H5Screate_simple(rank,dims,NULL))<0) + return -1; + + /* Create a dataset */ + if ((dset_id = H5Dcreate(loc_id,dset_name,type_id,space_id,H5P_DEFAULT))<0) + return -1; + + /* Write the buf */ + if ( buf ) + if (H5Dwrite(dset_id,type_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + return -1; + + /* Close */ + if (H5Dclose(dset_id)<0) + return -1; + if (H5Sclose(space_id)<0) + return -1; + + return 0; + +} + + +/*------------------------------------------------------------------------- + * Function: write_attr + * + * Purpose: utility function to write an attribute in LOC_ID + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: November 12, 2003 + * + *------------------------------------------------------------------------- + */ + +int make_attr(hid_t loc_id, + int rank, + hsize_t *dims, + const char *attr_name, + hid_t type_id, + void *buf) +{ + hid_t attr_id; + hid_t space_id; + + /* create a space */ + if ((space_id = H5Screate_simple(rank,dims,NULL))<0) + return -1; + + /* create the attribute */ + if ((attr_id = H5Acreate(loc_id,attr_name,type_id,space_id,H5P_DEFAULT))<0) + goto out; + + /* write the buffer */ + if ( buf ) + { + if (H5Awrite(attr_id,type_id,buf)<0) + goto out; + } + + /* close */ + H5Aclose(attr_id); + H5Sclose(space_id); + return 0; + +out: + H5E_BEGIN_TRY { + H5Aclose(attr_id); + H5Sclose(space_id); + } H5E_END_TRY; + return -1; +} + |