From d28bc61b335e447691521d37dbd8f58ebf83d9a1 Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Wed, 22 Oct 2003 16:43:34 -0500 Subject: [svn-r7703] Purpose: h5repack new features Description: implemented some features of h5repack , mostly borrowed from hrepack like 1) the parsing of options 2) building a list of options 3) printing formats added a test program that generates files and calls h5diff to compare the input and output files Platforms tested: Misc. update: --- tools/h5repack/Makefile.in | 16 +- tools/h5repack/h5repack.c | 407 +++++++++++++++++++++++++++++++++++- tools/h5repack/h5repack_list.c | 123 +++++++++++ tools/h5repack/h5repack_main.c | 24 +++ tools/h5repack/h5repack_opttable.c | 294 ++++++++++++++++++++++++++ tools/h5repack/h5repack_parse.c | 385 ++++++++++++++++++++++++++++++++++ tools/h5repack/h5repacktst.c | 107 ---------- tools/h5repack/test_h5repack_add.c | 195 +++++++++++++++++ tools/h5repack/test_h5repack_main.c | 117 +++++++++++ 9 files changed, 1553 insertions(+), 115 deletions(-) create mode 100644 tools/h5repack/h5repack_list.c create mode 100644 tools/h5repack/h5repack_main.c create mode 100644 tools/h5repack/h5repack_opttable.c create mode 100644 tools/h5repack/h5repack_parse.c delete mode 100644 tools/h5repack/h5repacktst.c create mode 100644 tools/h5repack/test_h5repack_add.c create mode 100644 tools/h5repack/test_h5repack_main.c diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in index e2fe63a..8768d36 100644 --- a/tools/h5repack/Makefile.in +++ b/tools/h5repack/Makefile.in @@ -21,9 +21,11 @@ srcdir=@srcdir@ ## Add include directory to the C preprocessor flags, add -lh5tools and ## -lhdf5 to the list of libraries. ## Add also the test include directory for #defines +## Add include for H5DIFF CPPFLAGS=-I. -I$(srcdir) -I$(top_builddir)/src -I$(top_srcdir)/src \ -I$(top_srcdir)/test \ - -I$(top_srcdir)/tools/lib @CPPFLAGS@ + -I$(top_srcdir)/tools/lib @CPPFLAGS@ \ + -I$(top_srcdir)/tools/h5diff ## Test programs and scripts. ## @@ -49,8 +51,10 @@ MOSTLYCLEAN=*.h5 ## Source and object files for programs... ## -PROG_SRC=h5repack.c h5repacktst.c +PROG_SRC=h5repack.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c ../h5diff/h5trav.c ../h5diff/h5trav_table.c test_h5repack_add.c test_h5repack_main.c PROG_OBJ=$(PROG_SRC:.c=.lo) +OBJS=h5repack.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo ../h5diff/h5trav.lo ../h5diff/h5trav_table.lo +TEST_OBJS=../h5diff/h5diff.lo ../h5diff/h5diff_array.lo ../h5diff/h5diff_dset.lo ../h5diff/h5diff_util.lo h5repack.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo ../h5diff/h5trav.lo ../h5diff/h5trav_table.lo test_h5repack_add.lo test_h5repack_main.lo PRIVATE_HDR= @@ -68,11 +72,11 @@ check test _test: $(PROGS) ## $(PROGS): $(LIBHDF5) -h5repack: h5repack.lo - @$(LT_LINK_EXE) $(CFLAGS) -o $@ h5repack.lo $(LIBHDF5) $(LDFLAGS) $(LIBS) +h5repack: $(OBJS) + @$(LT_LINK_EXE) $(CFLAGS) -o $@ $(OBJS) $(LIBHDF5) $(LDFLAGS) $(LIBS) -h5repacktst: h5repacktst.lo - @$(LT_LINK_EXE) $(CFLAGS) -o $@ h5repacktst.lo $(LIBHDF5) $(LDFLAGS) $(LIBS) +h5repacktst: $(TEST_OBJS) + @$(LT_LINK_EXE) $(CFLAGS) -o $@ $(TEST_OBJS) $(LIBHDF5) $(LDFLAGS) $(LIBS) @CONCLUDE@ diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index b6fd6f1..ea2b21a 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -12,12 +12,415 @@ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "hdf5.h" +#include +#include "h5repack.h" +#include "h5repack_opttable.h" +#include "h5repack_parse.h" +#include "h5repack_list.h" -int main (void) +static void print_options(packoptions_t *options); + +/*------------------------------------------------------------------------- + * Function: h5repack + * + * Purpose: locate all high-level HDF5 objects in the file + * and compress/chunk them using options + * + * Algorythm: 2 traversals are made to the file; the 1st builds a list of + * the objects, the 2nd makes a copy of them, using the options; + * the reason for the 1st traversal is to check for invalid + * object name requests + * + * Return: 0, ok, -1, fail + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +int h5repack(char* infile, + char* outfile, + packoptions_t *options) { + options->trip=0; + + /* also checks input */ + print_options(options); + /* first check for objects in input that are in the file */ + if (get_objlist(infile,options)==0) + { + /* the real deal now */ + options->trip=1; + + if (options->verbose) + printf("Making file <%s>...\n",outfile); + + } return 0; } + + + +/*------------------------------------------------------------------------- + * Function: h5repack_init + * + * Purpose: initialize options + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_init (packoptions_t *options, + int verbose) +{ + memset(options,0,sizeof(packoptions_t)); + options->threshold = 1024; + options->verbose = verbose; + return (options_table_init(&(options->op_tbl))); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_end + * + * Purpose: free options table + * + *------------------------------------------------------------------------- + */ + +int h5repack_end (packoptions_t *options) +{ + return options_table_free(options->op_tbl); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_addcomp + * + * Purpose: add a compression -t option to table + * Example: -t "*:GZIP 6" , STR = "*:GZIP 6" + * + * Return: 0, ok, exit, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_addcomp(char* str, + packoptions_t *options) +{ + obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ + comp_info_t comp; /*compression info for the current -t option entry */ + int n_objs; /*number of objects in the current -t or -c option entry */ + int i; + + if (options->all_comp==1){ + printf("Error: Invalid compression input: '*' is present with other objects <%s>\n",str); + exit(1); + } + + /* parse the -t option */ + obj_list=parse_comp(str,&n_objs,&comp); + + /* searh for the "*" all objects character */ + for (i = 0; i < n_objs; i++) + { + if (strcmp("*",obj_list[i].obj)==0) + { + /* if we are compressing all set the global comp type */ + options->all_comp=1; + options->comp_g=comp; + } + } + + if (i>1) + { + printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str); + free(obj_list); + options_table_free(options->op_tbl); + exit(1); + } + + if (options->all_comp==0) + options_add_comp(obj_list,n_objs,comp,options->op_tbl); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: h5repack_addchunk + * + * Purpose: add a chunk -c option to table + * Example: -c "*:2x2" , STR = "*:2x2" + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + + +int h5repack_addchunk(char* str, + packoptions_t *options) +{ + + obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ + int n_objs; /*number of objects in the current -t or -c option entry */ + hsize_t chunk_lengths[MAX_VAR_DIMS]; /* chunk lengths along each dimension */ + int chunk_rank; /*global rank for chunks */ + int i, j; + + if (options->all_chunk==1){ + printf("Error: Invalid chunking input: '*' is present with other objects <%s>\n",str); + exit(1); + } + + /* parse the -c option */ + obj_list=parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank); + + /* searh for the "*" all objects character */ + for (i = 0; i < n_objs; i++) + { + if (strcmp("*",obj_list[i].obj)==0) + { + /* if we are chunking all set the global chunking type */ + options->all_chunk=1; + options->chunk_g.rank=chunk_rank; + for (j = 0; j < chunk_rank; j++) + options->chunk_g.chunk_lengths[j] = chunk_lengths[j]; + } + } + + if (i>1) + { + printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str); + free(obj_list); + options_table_free(options->op_tbl); + exit(1); + } + + if (options->all_chunk==0) + options_add_chunk(obj_list,n_objs,chunk_lengths,chunk_rank,options->op_tbl); + + free(obj_list); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: print_options + * + * Purpose: print options, checks for invalid options + * + * Return: void, exit on error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +static +void print_options(packoptions_t *options) +{ + int i, k, j, has_cp=0, has_ck=0; + +/*------------------------------------------------------------------------- + * objects to chunk + *------------------------------------------------------------------------- + */ + if (options->verbose) + { + printf("\n"); + printf("Objects to chunk are...\n"); + if (options->all_chunk==1) { + printf("\tChunk all with dimension ["); + for ( j = 0; j < options->chunk_g.rank; j++) + printf("%d ", options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } + }/* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* obj_name=options->op_tbl->objs[i].path; + + if (options->op_tbl->objs[i].chunk.rank>0) + { + if (options->verbose){ + printf("\t<%s> with chunk size ",obj_name); + for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) + printf("%d ",options->op_tbl->objs[i].chunk.chunk_lengths[k]); + printf("\n"); + } + has_ck=1; + } + else if (options->op_tbl->objs[i].chunk.rank==-2) + { + if (options->verbose) + printf("\t%s %s\n",obj_name,"NONE"); + has_ck=1; + } + } + + if (options->all_chunk==1 && has_ck){ + printf("Error: Invalid chunking input: '*' is present with other objects\n"); + exit(1); + } + +/*------------------------------------------------------------------------- + * objects to compress/uncompress + *------------------------------------------------------------------------- + */ + + if (options->verbose) + { + printf("Objects to compress are...\n"); + if (options->all_comp==1) + { + switch (options->comp_g.type) + { + case H5Z_FILTER_NONE: + printf("\tUncompress all %s\n", + get_scomp(options->comp_g.type)); + break; + case H5Z_FILTER_SZIP: + printf("\tCompress all with %s compression\n", + get_scomp(options->comp_g.type)); + break; + case H5Z_FILTER_DEFLATE: + printf("\tCompress all with %s compression, parameter %d\n", + get_scomp(options->comp_g.type), + options->comp_g.info); + break; + }; + } + } /* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + pack_info_t obj=options->op_tbl->objs[i]; + if (obj.comp.type>0) + { + char* obj_name=obj.path; + if (options->verbose) { + printf("\t<%s> with %s compression, parameter %d\n", + obj_name, + get_scomp(obj.comp.type), + obj.comp.info); + } + has_cp=1; + } + } + + if (options->all_comp==1 && has_cp){ + printf("Error: Invalid compression input: * is present with other objects\n"); + exit(1); + } +} + +/*------------------------------------------------------------------------- + * Function: read_info + * + * Purpose: read comp and chunk options from file + * + * Return: void, exit on error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ + +void read_info(char *filename,packoptions_t *options) +{ + char stype[10]; + char comp_info[1024]; + FILE *fp; + char c; + int i, rc=1; + + + if ((fp = fopen(filename, "r")) == (FILE *)NULL) { + printf( "Cannot open options file %s", filename); + exit(1); + } + + /* Cycle until end of file reached */ + while( 1 ) + { + rc=fscanf(fp, "%s", stype); + if (rc==-1) + break; + + /*------------------------------------------------------------------------- + * comp + *------------------------------------------------------------------------- + */ + if (strcmp(stype,"-t") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!='"' ) + { + fscanf(fp, "%c", &c); + } + c='0'; + /* go until end */ + while( c!='"' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addcomp(comp_info,options)==-1){ + printf( "Could not add compression option. Exiting\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * chunk + *------------------------------------------------------------------------- + */ + else if (strcmp(stype,"-c") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!='"' ) + { + fscanf(fp, "%c", &c); + } + c='0'; + /* go until end */ + while( c!='"' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addchunk(comp_info,options)==-1){ + printf( "Could not add chunck option. Exiting\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * not valid + *------------------------------------------------------------------------- + */ + else { + printf( "Bad file format for %s", filename); + exit(1); + } + } + + fclose(fp); + return; +} + + diff --git a/tools/h5repack/h5repack_list.c b/tools/h5repack/h5repack_list.c new file mode 100644 index 0000000..21ec337e --- /dev/null +++ b/tools/h5repack/h5repack_list.c @@ -0,0 +1,123 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include +#include +#include +#include "h5repack.h" +#include "h5repack_list.h" +#include "h5diff.h" + + +/*------------------------------------------------------------------------- + * Function: get_objlist + * + * Purpose: locate all HDF5 objects in the file + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +int get_objlist(char* fname, packoptions_t *options) +{ + hid_t fid; + int nobjects, i; + info_t *info=NULL; + +/*------------------------------------------------------------------------- + * open the file + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY { + + /* Open the files */ + if ((fid=H5Fopen(fname,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ){ + printf("h5repack: <%s>: No such file or directory\n", fname ); + exit(1); + } + /* enable error reporting */ + } H5E_END_TRY; + + +/*------------------------------------------------------------------------- + * get the number of objects in the file + *------------------------------------------------------------------------- + */ + + if ((nobjects = H5get_object_info(fid, NULL ))<0) { + printf("h5repack: <%s>: Could not obtain object list\n", fname ); + return -1; + } + +/*------------------------------------------------------------------------- + * get the list of objects in the file + *------------------------------------------------------------------------- + */ + + if ((info = (info_t*) malloc( nobjects * sizeof(info_t)))==NULL){ + printf("h5repack: <%s>: Could not allocate object list\n", fname ); + return -1; + } + if (H5get_object_info(fid, info )<0) { + printf("h5repack: <%s>: Could not obtain object list\n", fname ); + return -1; + } + +/*------------------------------------------------------------------------- + * compare with user supplied list + *------------------------------------------------------------------------- + */ + + if (options->verbose) + printf("Opening file <%s>. Searching for objects to modify...\n",fname); + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* obj_name=options->op_tbl->objs[i].path; + if (options->verbose) + printf(PFORMAT1,"","",obj_name); + + /* the input object names are present in the file and are valid */ + if (info_getindex(obj_name,nobjects,info)<0) + { + printf("\nError: Could not find <%s> in file <%s>. Exiting...\n", + obj_name,fname); + H5Fclose(fid); + info_free(info,nobjects); + exit(1); + } + if (options->verbose) + printf("...Found\n"); + } + + + +/*------------------------------------------------------------------------- + * free + *------------------------------------------------------------------------- + */ + H5Fclose(fid); + info_free(info,nobjects); + return 0; + +} diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c new file mode 100644 index 0000000..b0415e4 --- /dev/null +++ b/tools/h5repack/h5repack_main.c @@ -0,0 +1,24 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" + + +int main (void) +{ + + + return 0; +} + diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c new file mode 100644 index 0000000..ef78ecb --- /dev/null +++ b/tools/h5repack/h5repack_opttable.c @@ -0,0 +1,294 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include +#include "h5repack_opttable.h" + + + +/*------------------------------------------------------------------------- + * Function: options_table_init + * + * Purpose: init options table + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int options_table_init( options_table_t **tbl ) +{ + int i; + options_table_t* table = (options_table_t*) malloc(sizeof(options_table_t)); + if (table==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + + table->size = 3; + table->nelems = 0; + table->objs = (pack_info_t*) malloc(table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + + for (i = 0; i < table->size; i++) { + strcpy(table->objs[i].path,"\0"); + table->objs[i].comp.info = -1; + table->objs[i].comp.type = -1; + table->objs[i].chunk.rank = -1; + } + + *tbl = table; + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_table_free + * + * Purpose: free table memory + * + * Return: 0 + * + *------------------------------------------------------------------------- + */ + +int options_table_free( options_table_t *table ) +{ + free(table->objs); + free(table); + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_add_chunk + * + * Purpose: add a chunking -c option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + + +int options_add_chunk( obj_list_t *obj_list, + int n_objs, + hsize_t *chunk_lengths, + int chunk_rank, + options_table_t *table ) +{ + int i, j, k, I, added=0, found=0; + + if (table->nelems+n_objs >= table->size) { + table->size += n_objs; + table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + for (i = table->nelems; i < table->size; i++) { + strcpy(table->objs[i].path,"\0"); + table->objs[i].comp.info = -1; + table->objs[i].comp.type = -1; + table->objs[i].chunk.rank = -1; + } + } + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* already chunk info inserted for this one; exit */ + if (table->objs[i].chunk.rank>0) + { + printf("Input Error: chunk information already inserted for <%s>\n",obj_list[j].obj); + exit(1); + } + /* insert the chunk info */ + else + { + table->objs[i].chunk.rank = chunk_rank; + for (k = 0; k < chunk_rank; k++) + table->objs[i].chunk.chunk_lengths[k] = chunk_lengths[k]; + found=1; + break; + } + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].chunk.rank = chunk_rank; + for (k = 0; k < chunk_rank; k++) + table->objs[I].chunk.chunk_lengths[k] = chunk_lengths[k]; + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].chunk.rank = chunk_rank; + for (k = 0; k < chunk_rank; k++) + table->objs[I].chunk.chunk_lengths[k] = chunk_lengths[k]; + } + } + + table->nelems+= added; + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: options_add_comp + * + * Purpose: add a compression -t option to the option list + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int options_add_comp( obj_list_t *obj_list, + int n_objs, + comp_info_t comp, + options_table_t *table ) +{ + + int i, j, I, added=0, found=0; + + if (table->nelems+n_objs >= table->size) { + table->size += n_objs; + table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + printf("Error: not enough memory for options table\n"); + return -1; + } + for (i = table->nelems; i < table->size; i++) { + strcpy(table->objs[i].path,"\0"); + table->objs[i].comp.info = -1; + table->objs[i].comp.type = -1; + table->objs[i].chunk.rank = -1; + } + } + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* already COMP info inserted for this one; exit */ + if (table->objs[i].comp.type>0) + { + printf("Input Error: compression information already inserted for <%s>\n",obj_list[j].obj); + exit(1); + } + /* insert the comp info */ + else + { + table->objs[i].comp = comp; + found=1; + break; + } + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].comp = comp; + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + table->objs[I].comp = comp; + } + } + + table->nelems+= added; + + return 0; +} + +/*------------------------------------------------------------------------- + * Function: options_get_object + * + * Purpose: get object from table; "path" is the key + * + * Return: pack_info_t* OBJECT or NULL if not found; PATH is the key + * + *------------------------------------------------------------------------- + */ + +pack_info_t* options_get_object( char *path, + options_table_t *table ) +{ + int i; + + for ( i = 0; i < table->nelems; i++) + { + /* found it */ + if (strcmp(table->objs[i].path,path)==0) + { + return (&table->objs[i]); + } + } + + return NULL; +} + + + + diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c new file mode 100644 index 0000000..36f6884 --- /dev/null +++ b/tools/h5repack/h5repack_parse.c @@ -0,0 +1,385 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include +#include "h5repack_parse.h" + + +/*------------------------------------------------------------------------- + * Function: parse_comp + * + * Purpose: read compression info + * + * Return: a list of names, the number of names and its compression type + * + * Examples: + * "AA,B,CDE:RLE" + * "*:GZIP 6" + * "A,B:NONE" + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +obj_list_t* parse_comp(char *str, int *n_objs, comp_info_t *comp) +{ + unsigned i, u; + char c; + size_t len=strlen(str); + int j, m, n, k, end_obj=-1, no_param=0; + char obj[MAX_NC_NAME]; + char scomp[10]; + char stype[5]; + obj_list_t* obj_list=NULL; + + /* initialize compression info */ + memset(comp,0,sizeof(comp_info_t)); + + /* check for the end of object list and number of objects */ + for ( i=0, n=0; i\n",str); + exit(1); + } + + n++; + obj_list=malloc(n*sizeof(obj_list_t)); + *n_objs=n; + + /* get object list */ + for ( j=0, k=0, n=0; j\n",str); + exit(1); + } + + + /* get compression type */ + m=0; + for ( i=end_obj+1, k=0; i\n",str); + exit(1); + } + stype[m]=c; + } + stype[m]='\0'; + comp->info=atoi(stype); + i+=m; /* jump */ + } + else if (i==len-1) { /*no more parameters */ + scomp[k+1]='\0'; + no_param=1; + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_NONE + *------------------------------------------------------------------------- + */ + if (strcmp(scomp,"NONE")==0) + comp->type=H5Z_FILTER_NONE; + +/*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"GZIP")==0) + { + comp->type=H5Z_FILTER_DEFLATE; + if (no_param) { /*no more parameters, GZIP must have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Missing compression parameter in <%s>\n",str); + exit(1); + } + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SZIP")==0) + { + comp->type=H5Z_FILTER_SZIP; + if (m>0){ /*SZIP does not have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Extra compression parameter in SZIP <%s>\n",str); + exit(1); + } + } + else { + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression type in <%s>\n",str); + exit(1); + } + } + } /*i*/ + + + /* check valid parameters */ + switch (comp->type) + { + case H5Z_FILTER_DEFLATE: + if (comp->info<0 || comp->info>9 ){ + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + case H5Z_FILTER_SZIP: + break; + }; + + return obj_list; +} + + +/*------------------------------------------------------------------------- + * Function: get_scomp + * + * Purpose: return the compression type as a string + * + * Return: name of filter, exit on error + * + *------------------------------------------------------------------------- + */ + +char* get_scomp(int code) +{ + if (code==H5Z_FILTER_NONE) + return "NONE"; + else if (code==H5Z_FILTER_DEFLATE) + return "GZIP"; + else if (code==H5Z_FILTER_SZIP) + return "SZIP"; + else { + printf("Input Error in compression type\n"); + exit(1); + } + return NULL; +} + + +/*------------------------------------------------------------------------- + * Function: parse_chunk + * + * Purpose: read chunkink info + * + * Return: a list of names, the number of names and its chunking info + * + * Examples: + * "AA,B,CDE:10X10 + * "*:10X10" + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +obj_list_t* parse_chunk(char *str, + int *n_objs, + hsize_t *chunk_lengths, + int *chunk_rank) +{ + obj_list_t* obj_list=NULL; + unsigned i; + char c; + size_t len=strlen(str); + int j, n, k, end_obj=-1, c_index; + char obj[MAX_NC_NAME]; + char sdim[10]; + + /* check for the end of object list and number of objects */ + for ( i=0, n=0; i\n",str); + exit(1); + } + + n++; + obj_list=malloc(n*sizeof(obj_list_t)); + *n_objs=n; + + /* get object list */ + for ( j=0, k=0, n=0; j\n",str); + exit(1); + } + + /* get chunk info */ + k=0; + for ( i=end_obj+1, c_index=0; i\n",str); + exit(1); + } + + if ( c=='x' || i==len-1) + { + if ( c=='x') { + sdim[k-1]='\0'; + k=0; + chunk_lengths[c_index]=atoi(sdim); + if (chunk_lengths[c_index]==0) { + if (obj_list) free(obj_list); + printf("Input Error: Invalid chunking in <%s>\n",str); + exit(1); + } + c_index++; + } + else if (i==len-1) { /*no more parameters */ + sdim[k]='\0'; + k=0; + if (strcmp(sdim,"NONE")==0) + { + *chunk_rank=-2; + } + else + { + chunk_lengths[c_index]=atoi(sdim); + if (chunk_lengths[c_index]==0){ + if (obj_list) free(obj_list); + printf("Input Error: Invalid chunking in <%s>\n",str); + exit(1); + } + *chunk_rank=c_index+1; + } + } /*if */ + } /*if c=='x' || i==len-1 */ + } /*i*/ + + return obj_list; +} + + + +/*------------------------------------------------------------------------- + * Function: parse_number + * + * Purpose: read a number from command line argument + * + * Return: number, -1 for FAIL + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ + + +int parse_number(char *str) +{ + unsigned i; + int n; + char c; + size_t len=strlen(str); + + for ( i=0; i - * September, 19, 2003 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -static int -test() -{ - hid_t fid; /* File ID */ - hid_t fapl; /* File access property list */ - hid_t dsid; /* Dataset ID */ - hid_t sid; /* Dataspace ID */ - - TESTING(" h5repack"); - - if((fid = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))<0) - TEST_ERROR; - - if((sid = H5Screate (H5S_SCALAR))<0) - TEST_ERROR; - - if((dsid = H5Dcreate (fid, "dset1", H5T_NATIVE_INT, sid, H5P_DEFAULT))<0) - TEST_ERROR; - - if(H5Dclose(dsid)<0) - TEST_ERROR; - - if(H5Fclose(fid)<0) - TEST_ERROR; - - PASSED(); - return 0; - -error: - return 1; -} - - -/*------------------------------------------------------------------------- - * Function: main - * - * Purpose: Executes h5repack tests - * - * Return: Success: zero - * Failure: non-zero - * - * Programmer: Pedro Vicente - * September, 19, 2003 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ - -int main (void) -{ - int nerrors=0; - - /* run tests */ - puts("Testing h5repack:"); - nerrors += test(); - - - /* check for errors */ - if (nerrors) - goto error; - puts("All h5repack passed."); - - return 0; - -error: - puts("***** H5REPACK TESTS FAILED *****"); - return 1; - - - return 0; -} \ No newline at end of file diff --git a/tools/h5repack/test_h5repack_add.c b/tools/h5repack/test_h5repack_add.c new file mode 100644 index 0000000..5edab2b --- /dev/null +++ b/tools/h5repack/test_h5repack_add.c @@ -0,0 +1,195 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "test_h5repack_add.h" + + +/*------------------------------------------------------------------------- + * Function: make_deflate + * + * Purpose: make a dataset using DEFLATE (GZIP) compression + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * September, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +make_deflate(hid_t fid) +{ + hid_t dcpl; /* dataset creation property list */ + hid_t dsid; /* dataset ID */ + hid_t sid; /* dataspace ID */ + int rank=2; + hsize_t dims[2]={4,2}; + hsize_t chunk_dims[2]={2,1}; + int buf[4][2]={1,2,3,4,5,6,7,8}; + + /* create a space */ + if((sid = H5Screate_simple(rank, dims, NULL))<0) + TEST_ERROR; + + /* create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + TEST_ERROR; + + /* set up for deflated data */ + if(H5Pset_chunk(dcpl, rank, chunk_dims)<0) + TEST_ERROR; + if(H5Pset_deflate(dcpl, 9)<0) + TEST_ERROR; + + /* create the dataset */ + if((dsid = H5Dcreate (fid, "dset1", H5T_NATIVE_INT, sid, dcpl))<0) + TEST_ERROR; + + /* write the data to the dataset */ + if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)<0) + TEST_ERROR; + + /* close */ + if(H5Dclose(dsid)<0) + TEST_ERROR; + if(H5Pclose(dcpl)<0) + TEST_ERROR; + if(H5Sclose(sid)<0) + TEST_ERROR; + + return 0; + +error: + return 1; +} + +/*------------------------------------------------------------------------- + * Function: make_szip + * + * Purpose: make a dataset using SZIP compression + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * September, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +make_szip(hid_t fid) +{ + hid_t dcpl; /* dataset creation property list */ + hid_t dsid; /* dataset ID */ + hid_t sid; /* dataspace ID */ + int rank=2; + hsize_t dims[2]={4,2}; + hsize_t chunk_dims[2]={2,1}; + int buf[4][2]={1,2,3,4,5,6,7,8}; + unsigned szip_options_mask=H5_SZIP_ALLOW_K13_OPTION_MASK|H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; + + /* + pixels_per_block must be an even number, and <= pixels_per_scanline + and <= MAX_PIXELS_PER_BLOCK + */ + szip_pixels_per_block=16; + + /* create a space */ + if((sid = H5Screate_simple(rank, dims, NULL))<0) + TEST_ERROR; + + /* create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + TEST_ERROR; + + /* set up for sziped data */ + if(H5Pset_chunk(dcpl, rank, chunk_dims)<0) + TEST_ERROR; + if(H5Pset_szip (dcpl, szip_options_mask, szip_pixels_per_block)<0) + TEST_ERROR; + + /* create the dataset */ + if((dsid = H5Dcreate (fid, "dset2", H5T_NATIVE_INT, sid, dcpl))<0) + TEST_ERROR; + + /* write the data to the dataset */ + if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)<0) + TEST_ERROR; + + /* close */ + if(H5Dclose(dsid)<0) + TEST_ERROR; + if(H5Pclose(dcpl)<0) + TEST_ERROR; + if(H5Sclose(sid)<0) + TEST_ERROR; + + return 0; + +error: + return 1; +} + + +/*------------------------------------------------------------------------- + * Function: make_dsets + * + * Purpose: make several datasets with DEFLATE and SZIP filters + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * September, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +int make_dsets() +{ + hid_t fid; /* file ID */ + int nerrors=0; + + TESTING(" generating datasets"); + + /* create a file */ + if((fid = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))<0) + TEST_ERROR; + + nerrors += make_deflate(fid); + nerrors += make_szip(fid); + + /* close */ + if(H5Fclose(fid)<0) + TEST_ERROR; + + if (nerrors) + goto error; + + PASSED(); + return 0; + +error: + return 1; +} + diff --git a/tools/h5repack/test_h5repack_main.c b/tools/h5repack/test_h5repack_main.c new file mode 100644 index 0000000..1768609 --- /dev/null +++ b/tools/h5repack/test_h5repack_main.c @@ -0,0 +1,117 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" +#include "h5diff.h" +#include "test_h5repack_add.h" + + + + +/*------------------------------------------------------------------------- + * Function: test + * + * Purpose: + * + * 1) compress/chunk FILENAME with some compression/chunking options + * 2) use the h5diff utility to compare the input and output file; + * it returns RET==0 if the objects have the same data + * 3) use API functions to verify the compression/chunking input on the otput file + * + * Return: Success: zero + * Failure: 1 + * + * Programmer: Pedro Vicente + * September, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +test() +{ + packoptions_t options; + options_t diff_options; + memset(&diff_options, 0, sizeof (options_t)); + + + TESTING(" deflate filter"); + + if (h5repack_init (&options, 0)<0) + TEST_ERROR; + if (h5repack_addcomp("dset2:GZIP 9",&options)<0) + TEST_ERROR; + if (h5repack_addchunk("dset2:5x4",&options)<0) + TEST_ERROR; + if (h5repack(FILENAME,FILENAME_OUT,&options)<0) + TEST_ERROR; + if (h5repack_end (&options)<0) + TEST_ERROR; + if (h5diff(FILENAME,FILENAME,NULL,NULL,diff_options) == 1) + TEST_ERROR; + + PASSED(); + return 0; + + +error: + return 1; + +} + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Executes h5repack tests + * + * Return: Success: zero + * Failure: non-zero + * + * Programmer: Pedro Vicente + * September, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +int main (void) +{ + int nerrors=0; + + /* run tests */ + puts("Testing h5repack:"); + nerrors += make_dsets(); + nerrors += test(); + + + /* check for errors */ + if (nerrors) + goto error; + puts("All h5repack tests passed."); + + return 0; + +error: + puts("***** H5REPACK TESTS FAILED *****"); + return 1; + + + return 0; +} + -- cgit v0.12