diff options
Diffstat (limited to 'tools/h5repack')
| -rw-r--r-- | tools/h5repack/h5repack.c | 13 | ||||
| -rw-r--r-- | tools/h5repack/h5repack.h | 39 | ||||
| -rwxr-xr-x | tools/h5repack/h5repack.sh.in | 14 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_copy.c | 149 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_filters.c | 35 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_main.c | 55 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_opttable.c | 31 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_parse.c | 87 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_refs.c | 13 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_verify.c | 19 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_attr.c | 31 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_dset.c | 26 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_main.c | 73 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_make.c | 67 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_util.c | 6 |
15 files changed, 424 insertions, 234 deletions
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 7ba5a6c..f9cb295 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -234,9 +234,6 @@ static int check_options(pack_opt_t *options) { int i, k, j, has_cp=0, has_ck=0; - unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; - unsigned szip_pixels_per_block; - /*------------------------------------------------------------------------- * objects to layout *------------------------------------------------------------------------- @@ -337,8 +334,17 @@ static int check_options(pack_opt_t *options) default: break; case H5Z_FILTER_SZIP: + { + + unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; szip_pixels_per_block=pack.filter[j].cd_values[0]; + + if (pack.filter[j].szip_coding==0) + szip_options_mask=H5_SZIP_NN_OPTION_MASK; + else + szip_options_mask=H5_SZIP_EC_OPTION_MASK; /* check szip parameters */ if ( pack.chunk.rank!=-1 /* @@ -364,6 +370,7 @@ static int check_options(pack_opt_t *options) } + } break; } /* switch */ diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 3a3a7c0..4691a74 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -21,7 +21,9 @@ #include "h5diff.h" #include "h5tools.h" +#if 0 #define CHECK_SZIP +#endif #define H5FOPENERROR "unable to open file" @@ -58,6 +60,11 @@ typedef struct { typedef struct { H5Z_filter_t filtn; /* filter identification number */ int cd_values[CDVALUES]; /* filter client data values */ + /* extra input for szip, selects the coding method + entropy coding method: EC=0 + nearest neighbor coding method: NN=1 + */ + int szip_coding; } filter_info_t; /* chunk lengths along each dimension and rank */ @@ -192,17 +199,6 @@ int apply_filters(const char* name, /* object name from traverse list */ int has_filter(hid_t dcpl_id, H5Z_filter_t filtnin); -int check_szip_params( unsigned bits_per_pixel, - unsigned pixels_per_block, - unsigned pixels_per_scanline, - hsize_t image_pixels); - -int check_szip(hid_t type_id, /* dataset datatype */ - int rank, /* chunk rank */ - hsize_t *dims, /* chunk dims */ - unsigned szip_options_mask /*IN*/, - unsigned *szip_pixels_per_block /*IN,OUT*/, - pack_opt_t *options); int can_read(const char* name, /* object name from traverse list */ hid_t dcpl_id, /* dataset creation property list */ @@ -306,12 +302,12 @@ void write_attr_in(hid_t loc_id, const char* dset_name, /* for saving reference to dataset*/ hid_t fid, /* for reference create */ int make_diffs /* flag to modify data buffers */); -void write_null_attr(hid_t loc_id); void write_dset_in(hid_t loc_id, const char* dset_name, /* for saving reference to dataset*/ hid_t file_id, int make_diffs /* flag to modify data buffers */); -void write_null_dset(hid_t loc_id); + + /*------------------------------------------------------------------------- * tests utils @@ -337,6 +333,22 @@ int make_attr(hid_t loc_id, *------------------------------------------------------------------------- */ +#if defined (CHECK_SZIP) + +int check_szip_params( unsigned bits_per_pixel, + unsigned pixels_per_block, + unsigned pixels_per_scanline, + hsize_t image_pixels); + +int check_szip(hid_t type_id, /* dataset datatype */ + int rank, /* chunk rank */ + hsize_t *dims, /* chunk dims */ + unsigned szip_options_mask /*IN*/, + unsigned *szip_pixels_per_block /*IN,OUT*/, + pack_opt_t *options); + + + typedef struct { int compression_mode; @@ -368,6 +380,7 @@ typedef struct #define NN_MODE 1 #endif +#endif /* CHECK_SZIP */ #endif /* H5REPACK_H__ */ diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index e4d65de..98b46ba 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -25,7 +25,7 @@ USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@" H5REPACK=h5repack # The tool name H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary -H5DIFF=../h5diff/h5diff # The h5diff tool name +H5DIFF=../h5diff/h5diff # The h5diff tool name H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary nerrors=0 @@ -68,9 +68,9 @@ DIFFTEST() { VERIFY $@ if [ "`uname -s`" = "TFLOPS O/S" ]; then - $RUNSERIAL $H5DIFF_BIN $@ + $RUNSERIAL $H5DIFF_BIN $@ -q else - $RUNSERIAL $H5DIFF_BIN "$@" + $RUNSERIAL $H5DIFF_BIN "$@" -q fi RET=$? if [ $RET != 0 ] ; then @@ -151,7 +151,7 @@ else fi # szip with individual object -arg="test4.h5 -f dset2:SZIP=8 -l dset2:CHUNK=20x10" +arg="test4.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -159,7 +159,7 @@ else fi # szip for all -arg="test4.h5 -f SZIP=8" +arg="test4.h5 -f SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -200,7 +200,7 @@ else fi # all filters -arg="test4.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8 -f dset2:GZIP=1 -l dset2:CHUNK=20x10" +arg="test4.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -285,7 +285,7 @@ fi #filter conversions -arg="test_deflate.h5 -f dset_deflate:SZIP=8" +arg="test_deflate.h5 -f dset_deflate:SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 870f3c7..110069b 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -19,6 +19,110 @@ #include "H5private.h" #include "h5repack.h" +#if 0 +#define PRINT_DEBUG +#endif + +/*------------------------------------------------------------------------- + * Function: print_obj + * + * Purpose: print name and filters of an object + * + *------------------------------------------------------------------------- + */ +static void print_obj(hid_t dcpl_id, char *name) +{ + char str[255]; +#if defined (PRINT_DEBUG ) + char temp[255]; +#endif + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + int i; + + strcpy(str,"\0"); + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return; + + for ( i=0; i<nfilters; i++) + { + cd_nelmts = NELMTS(cd_values); + +#ifdef H5_WANT_H5_V1_6_COMPAT + filtn = H5Pget_filter(dcpl_id, + (unsigned)i, + &filt_flags, + &cd_nelmts, + cd_values, + sizeof(f_name), + f_name); +#else + filtn = H5Pget_filter(dcpl_id, + (unsigned)i, + &filt_flags, + &cd_nelmts, + cd_values, + sizeof(f_name), + f_name, + NULL); +#endif /* H5_WANT_H5_V1_6_COMPAT */ + + + switch (filtn) + { + default: + break; + case H5Z_FILTER_DEFLATE: + strcat(str,"GZIP "); + +#if defined (PRINT_DEBUG) + { + unsigned level=cd_values[0]; + sprintf(temp,"(%d)",level); + strcat(str,temp); + } +#endif + + break; + case H5Z_FILTER_SZIP: + strcat(str,"SZIP "); + +#if defined (PRINT_DEBUG) + { + unsigned options_mask=cd_values[0]; /* from dcpl, not filt*/ + unsigned ppb=cd_values[1]; + sprintf(temp,"(%d,",ppb); + strcat(str,temp); + if (options_mask & H5_SZIP_EC_OPTION_MASK) + strcpy(temp,"EC) "); + else if (options_mask & H5_SZIP_NN_OPTION_MASK) + strcpy(temp,"NN) "); + } + strcat(str,temp); + +#endif + + break; + case H5Z_FILTER_SHUFFLE: + strcat(str,"SHUF "); + break; + case H5Z_FILTER_FLETCHER32: + strcat(str,"FLET "); + break; + } /* switch */ + }/*i*/ + + printf(" %-10s %s %s\n", "dataset",str,name ); + +} + + /*------------------------------------------------------------------------- * Function: copy_objects * @@ -112,46 +216,6 @@ out: } -/*------------------------------------------------------------------------- - * Function: print_obj - * - * Purpose: print name and filters of an object - * - *------------------------------------------------------------------------- - */ - - -void print_obj(pack_info_t *obj, char *name) -{ - char str[26]; /*5x5+1*/ - int i; - - strcpy(str,"\0"); - - for ( i=0; i<obj->nfilters; i++) - { - switch (obj->filter[i].filtn) - { - default: - break; - case H5Z_FILTER_DEFLATE: - strcat(str,"GZIP "); - break; - case H5Z_FILTER_SZIP: - strcat(str,"SZIP "); - break; - case H5Z_FILTER_SHUFFLE: - strcat(str,"SHUF "); - break; - case H5Z_FILTER_FLETCHER32: - strcat(str,"FLET "); - break; - } /* switch */ - }/*i*/ - - printf(" %-10s %s %s\n", "dataset",str,name ); - -} /*------------------------------------------------------------------------- @@ -262,9 +326,6 @@ int do_copy_objects(hid_t fidin, if ((msize=H5Tget_size(mtype_id))==0) goto error; - if (options->verbose) - print_filters(dcpl_id); - /*------------------------------------------------------------------------- * check for external files *------------------------------------------------------------------------- @@ -370,7 +431,7 @@ int do_copy_objects(hid_t fidin, free(buf); if (options->verbose && wrote) - print_obj(&obj,travt->objs[i].name ); + print_obj(dcpl_id,travt->objs[i].name ); diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 61ef4a7..2536ced 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -122,8 +122,6 @@ int apply_filters(const char* name, /* object name from traverse list */ unsigned aggression; /* the deflate level */ hsize_t nelmts; /* number of elements in dataset */ size_t size; /* size of datatype in bytes */ - unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; - unsigned szip_pixels_per_block; int i; /* check first if the object is to be filtered */ @@ -205,15 +203,23 @@ int apply_filters(const char* name, /* object name from traverse list */ *------------------------------------------------------------------------- */ case H5Z_FILTER_SZIP: - szip_pixels_per_block=obj->filter[i].cd_values[0]; - + { + unsigned options_mask; + unsigned pixels_per_block; + + pixels_per_block=obj->filter[i].cd_values[0]; + if (obj->filter[i].szip_coding==0) + options_mask=H5_SZIP_NN_OPTION_MASK; + else + options_mask=H5_SZIP_EC_OPTION_MASK; + #if defined (CHECK_SZIP) /* check szip parameters */ if (check_szip(type_id, obj->chunk.rank, obj->chunk.chunk_lengths, - szip_options_mask, - &szip_pixels_per_block, + options_mask, + &pixels_per_block, options)==1) { #endif @@ -221,9 +227,10 @@ int apply_filters(const char* name, /* object name from traverse list */ /* set up for szip data */ if(H5Pset_chunk(dcpl_id,obj->chunk.rank,obj->chunk.chunk_lengths)<0) return -1; - if (H5Pset_szip(dcpl_id, szip_options_mask, szip_pixels_per_block)<0) + if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0) return -1; + #if defined (CHECK_SZIP) } else @@ -234,6 +241,7 @@ int apply_filters(const char* name, /* object name from traverse list */ #endif + } break; /*------------------------------------------------------------------------- @@ -269,7 +277,6 @@ out: } - /*------------------------------------------------------------------------- * Function: print_filters * @@ -277,7 +284,7 @@ out: * * Return: 0, ok, -1 no * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 19, 2003 * @@ -292,7 +299,7 @@ int print_filters(hid_t dcpl_id) unsigned cd_values[20]; /* filter client data values */ size_t cd_nelmts; /* filter client number of values */ size_t cd_num; /* filter client data counter */ - char f_name[256]; /* filter/file name */ + char f_name[256]; /* filter name */ char s[64]; /* temporary string buffer */ int i; @@ -338,6 +345,9 @@ int print_filters(hid_t dcpl_id) } +#if defined (CHECK_SZIP) + + /*------------------------------------------------------------------------- * Function: check_szip * @@ -353,7 +363,7 @@ int print_filters(hid_t dcpl_id) * Return: 1=can apply the filter * 0=cannot apply the filter * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 23, 2003 * @@ -560,3 +570,6 @@ int check_szip_params( unsigned bits_per_pixel, return 1; } +#endif /* CHECK_SZIP */ + + diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 810d30f..442af99 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -17,9 +17,25 @@ static void usage(void); -/* -h5repack main program -*/ +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: h5repack main program + * + * Return: 1, error, 0, no error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: May 9, 2003 + * + * Comments: + * + * Modifications: July 2004 + * Introduced the extra EC or NN option for SZIP + * + *------------------------------------------------------------------------- + */ + int main(int argc, char **argv) { @@ -33,6 +49,10 @@ int main(int argc, char **argv) for ( i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0) { + usage(); + exit(0); + } if (strcmp(argv[i], "-i") == 0) { infile = argv[++i]; } @@ -76,11 +96,15 @@ int main(int argc, char **argv) else if (argv[i][0] == '-') { usage(); + exit(1); } } if (infile == NULL || outfile == NULL) + { usage(); + exit(1); + } /* pack it */ ret=h5repack(infile,outfile,&options); @@ -112,8 +136,9 @@ void usage(void) printf("-i input Input HDF5 File\n"); printf("-o output Output HDF5 File\n"); printf("[-h] Print usage message\n"); + printf("[-v] Verbose mode. Print output (list of objects, filters, warnings)\n"); printf("[-f 'filter'] Filter type: 'filter' is a string with the format\n"); - printf(" <list of objects> : <name of filter> <filter parameters>\n"); + printf(" <list of objects> : <name of filter> = <filter parameters>\n"); printf(" <list of objects> is a comma separated list of object names\n"); printf(" meaning apply compression only to those objects.\n"); printf(" if no object names are specified, the filter is applied to all objects\n"); @@ -124,8 +149,10 @@ void usage(void) printf(" FLET, to apply the HDF5 checksum filter\n"); printf(" NONE, to remove the filter\n"); printf(" <filter parameters> is optional compression info\n"); - printf(" GZIP, the deflation level\n"); - printf(" SZIP, the pixels per block parameter\n"); + printf(" SHUF (no parameter)\n"); + printf(" FLET (no parameter)\n"); + printf(" GZIP=<deflation level> from 1-9\n"); + printf(" SZIP=<pixels per block,coding> (pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN')\n"); printf("[-l 'layout'] Layout type. 'layout' is a string with the format\n"); printf(" <list of objects> : <layout type>\n"); printf(" <list of objects> is a comma separated list of object names,\n"); @@ -143,8 +170,20 @@ void usage(void) printf("-m number Do not apply the filter to objects which size in bytes is smaller than number.\n"); printf(" If no size is specified a minimum of 1024 bytes is assumed.\n"); printf("\n"); - - exit(1); + printf("Examples of use:\n"); + printf("\n"); + printf("1) h5repack -i file1 -o file2 -f GZIP=1 -v\n"); + printf("\n"); + printf(" Applies GZIP compression to all objects in file1 and saves the output in file2\n"); + printf("\n"); + printf("2) h5repack -i file1 -o file2 -f dset1:SZIP=8,NN -v\n"); + printf("\n"); + printf(" Applies SZIP compression only to object 'dset1'\n"); + printf("\n"); + printf("3) h5repack -i file1 -o file2 -l dset1,dset2:CHUNK=20x10 -v\n"); + printf("\n"); + printf(" Applies chunked layout to objects 'dset1' and 'dset2'\n"); + printf("\n"); } diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 71cf1a8..cf76e82 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -35,7 +35,8 @@ void init_packobject(pack_info_t *obj) strcpy(obj->path,"\0"); for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) { - obj->filter[j].filtn = -1; + obj->filter[j].filtn = -1; + obj->filter[j].szip_coding = -1; for ( k=0; k<CDVALUES; k++) obj->filter[j].cd_values[k] = -1; } @@ -109,7 +110,7 @@ static void aux_tblinsert_layout(pack_opttbl_t *table, static int aux_inctable(pack_opttbl_t *table, int n_objs ) { - int i, j, k; + int i; table->size += n_objs; table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); @@ -119,17 +120,7 @@ static int aux_inctable(pack_opttbl_t *table, int n_objs ) } for (i = table->nelems; i < table->size; i++) { - strcpy(table->objs[i].path,"\0"); - for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) - { - table->objs[i].filter[j].filtn = -1; - for ( k=0; k<CDVALUES; k++) - table->objs[i].filter[j].cd_values[k] = -1; - } - table->objs[i].chunk.rank = -1; - table->objs[i].refobj_id = -1; - table->objs[i].layout = H5D_LAYOUT_ERROR; - table->objs[i].nfilters = 0; + init_packobject(&table->objs[i]); } return 0; } @@ -146,7 +137,7 @@ static int aux_inctable(pack_opttbl_t *table, int n_objs ) int options_table_init( pack_opttbl_t **tbl ) { - int i, j, k; + int i; pack_opttbl_t* table = (pack_opttbl_t*) malloc(sizeof(pack_opttbl_t)); if (table==NULL) { printf("Error: not enough memory for options table\n"); @@ -163,17 +154,7 @@ int options_table_init( pack_opttbl_t **tbl ) for ( i=0; i<table->size; i++) { - strcpy(table->objs[i].path,"\0"); - for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) - { - table->objs[i].filter[j].filtn = -1; - for ( k=0; k<CDVALUES; k++) - table->objs[i].filter[j].cd_values[k] = -1; - } - table->objs[i].chunk.rank = -1; - table->objs[i].refobj_id = -1; - table->objs[i].layout = H5D_LAYOUT_ERROR; - table->objs[i].nfilters = 0; + init_packobject(&table->objs[i]); } *tbl = table; diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index 96f7bce..e6974c0 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -56,10 +56,11 @@ obj_list_t* parse_filter(const char *str, unsigned i, u; char c; size_t len=strlen(str); - int j, m, n, k, end_obj=-1, no_param=0; + int j, m, n, k, l, end_obj=-1, no_param=0; char sobj[MAX_NC_NAME]; char scomp[10]; char stype[5]; + char smask[3]; obj_list_t* obj_list=NULL; unsigned pixels_per_block; @@ -132,17 +133,74 @@ obj_list_t* parse_filter(const char *str, if ( c=='=') { /*one more parameter */ scomp[k]='\0'; /*cut space */ - /* here we could have 1, 2 or 3 digits */ - for ( m=0,u=i+1; u<len; u++,m++) { - c = str[u]; - if (!isdigit(c)){ - if (obj_list) free(obj_list); - printf("Input Error: Compression parameter not digit in <%s>\n",str); - exit(1); - } - stype[m]=c; - } - stype[m]='\0'; + /*SZIP is a special case , it can be + SZIP=8,EC + SZIP=8,NN + */ + + if (strcmp(scomp,"SZIP")==0) + { + l=-1; /* mask index check */ + for ( m=0,u=i+1; u<len; u++,m++) + { + if (str[u]==',') + { + stype[m]='\0'; /* end digit of szip */ + l=0; /* start EC or NN search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!isdigit(c) && l==-1){ + if (obj_list) free(obj_list); + printf("Input Error: Compression parameter not digit in <%s>\n",str); + exit(1); + } + if (l==-1) + stype[m]=c; + else + { + smask[l]=c; + l++; + if (l==2) + { + smask[l]='\0'; + i=len-1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (strcmp(smask,"NN")==0) + filt->szip_coding=0; + else if (strcmp(smask,"EC")==0) + filt->szip_coding=1; + else + { + printf("Input Error: szip mask must be 'NN' or 'EC' \n"); + exit(1); + } + + } + } + + } /* u */ + } /*if */ + + else + { + /* here we could have 1 or 2 digits */ + for ( m=0,u=i+1; u<len; u++,m++) + { + c = str[u]; + if (!isdigit(c)){ + if (obj_list) free(obj_list); + printf("Input Error: Compression parameter not digit in <%s>\n",str); + exit(1); + } + stype[m]=c; + } /* u */ + + stype[m]='\0'; + } /*if */ + + + filt->cd_values[j++]=atoi(stype); i+=m; /* jump */ } @@ -249,6 +307,11 @@ obj_list_t* parse_filter(const char *str, printf("Input Error: pixels_per_block is too large in <%s>\n",str); exit(1); } + if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) { + if (obj_list) free(obj_list); + printf("Input Error: szip mask must be 'NN' or 'EC' \n"); + exit(1); + } break; }; diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index ef7f298..0db17f5 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -22,7 +22,7 @@ static const char* MapIdToName(hid_t refobj_id, trav_table_t *travt); -static void close_obj(H5G_obj_t obj_type, hid_t obj_id); +static void close_obj(H5G_obj_t1 obj_type, hid_t obj_id); static int copy_refs_attr(hid_t loc_in, @@ -142,7 +142,6 @@ int do_copy_refobjs(hid_t fidin, */ if (next==0 && h5tools_canreadf((NULL),dcpl_id)==1) { - /*------------------------------------------------------------------------- * test for a valid output dataset *------------------------------------------------------------------------- @@ -156,7 +155,7 @@ int do_copy_refobjs(hid_t fidin, */ if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) { - H5G_obj_t obj_type; + H5G_obj_t1 obj_type; hid_t refobj_id; hobj_ref_t *refbuf=NULL; /* buffer for object references */ hobj_ref_t *buf=NULL; @@ -231,7 +230,7 @@ int do_copy_refobjs(hid_t fidin, */ else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) { - H5G_obj_t obj_type; + H5G_obj_t1 obj_type; hid_t refobj_id; hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ hdset_reg_ref_t *buf=NULL; /* output buffer */ @@ -528,7 +527,7 @@ static int copy_refs_attr(hid_t loc_in, */ if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) { - H5G_obj_t obj_type; + H5G_obj_t1 obj_type; hid_t refobj_id; hobj_ref_t *refbuf=NULL; unsigned k; @@ -605,7 +604,7 @@ static int copy_refs_attr(hid_t loc_in, */ else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) { - H5G_obj_t obj_type; + H5G_obj_t1 obj_type; hid_t refobj_id; hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ hdset_reg_ref_t *buf=NULL; /* output buffer */ @@ -715,7 +714,7 @@ error: *------------------------------------------------------------------------- */ -static void close_obj(H5G_obj_t obj_type, hid_t obj_id) +static void close_obj(H5G_obj_t1 obj_type, hid_t obj_id) { H5E_BEGIN_TRY { diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index 30d3c05..983371e 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -25,7 +25,7 @@ * * Return: 1 has, 0 does not, -1 error * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 19, 2003 * @@ -99,7 +99,7 @@ int has_filter(hid_t dcpl_id, * * Return: 1 has, 0 does not, -1 error * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 30, 2003 * @@ -157,7 +157,7 @@ int has_layout(hid_t dcpl_id, * * Return: 1=filter present, 0=filter not present, -1=error * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 19, 2003 * @@ -193,11 +193,6 @@ int h5repack_verify(const char *fname, goto error; if ((dcpl_id=H5Dget_create_plist(dset_id))<0) goto error; - - if (options->verbose) { - printf(" %-10s %s\n", "dataset",name ); - print_filters(dcpl_id); - } /*------------------------------------------------------------------------- * filter check @@ -263,11 +258,7 @@ int h5repack_verify(const char *fname, goto error; if ((dcpl_id=H5Dget_create_plist(dset_id))<0) goto error; - - if (options->verbose) { - printf(" %-10s %s\n", "dataset",name ); - print_filters(dcpl_id); - } + /*------------------------------------------------------------------------- * filter check *------------------------------------------------------------------------- @@ -344,7 +335,7 @@ error: * * Return: 1=identical, 0=not identical, -1=error * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: December 31, 2003 * diff --git a/tools/h5repack/testh5repack_attr.c b/tools/h5repack/testh5repack_attr.c index dc03415..a1dad92 100644 --- a/tools/h5repack/testh5repack_attr.c +++ b/tools/h5repack/testh5repack_attr.c @@ -21,7 +21,7 @@ * * Purpose: write attributes in LOC_ID (dataset, group, named datatype) * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: November 12, 2003 * @@ -1028,30 +1028,5 @@ etc make_attr(loc_id,3,dims3,"float3D",H5T_NATIVE_FLOAT,buf83); } -/*------------------------------------------------------------------------- - * Function: write_null_attr - * - * Purpose: write null attribute in LOC_ID (dataset, group, named datatype) - * - * Programmer: Raymond Lu, slu@ncsa.uiuc.edu - * - * Date: May 24, 2004 - * - *------------------------------------------------------------------------- - */ -void write_null_attr(hid_t loc_id) -{ - hid_t sid, attr_id; - int val = 2; - - /* Create the null attribute */ - sid = H5Screate(H5S_NULL); - attr_id = H5Acreate(loc_id, "null_attr", H5T_NATIVE_INT, sid, H5P_DEFAULT); - - /* Not supposed to write anything */ - H5Awrite(attr_id, H5T_NATIVE_INT, &val); - - /* Close */ - H5Aclose(attr_id); - H5Sclose(sid); -} + + diff --git a/tools/h5repack/testh5repack_dset.c b/tools/h5repack/testh5repack_dset.c index 7f6345b..57a5b27 100644 --- a/tools/h5repack/testh5repack_dset.c +++ b/tools/h5repack/testh5repack_dset.c @@ -617,31 +617,7 @@ void write_dset_in(hid_t loc_id, write_dset(loc_id,3,dims3,"float3D",H5T_NATIVE_FLOAT,buf83); } -/*------------------------------------------------------------------------- - * Function: write_null_dset - * - * Purpose: write null dataset in LOC_ID - * - * Programmer: Raymond Lu, slu@ncsa.uiuc.edu - * - * Date: May 24, 2004 - * - *------------------------------------------------------------------------- - */ -void write_null_dset(hid_t loc_id) -{ - hid_t sid, dset_id; - int val = 2; - - /* Create the null dataset */ - sid = H5Screate(H5S_NULL); - dset_id = H5Dcreate(loc_id,"null_dset",H5T_NATIVE_INT,sid,H5P_DEFAULT); - H5Dwrite(dset_id,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val); - - /* Close */ - H5Dclose(dset_id); - H5Sclose(sid); -} + /*------------------------------------------------------------------------- * Function: make_dset_reg_ref diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 0c976f4..af19db2 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -29,7 +29,7 @@ * Return: Success: zero * Failure: non-zero * - * Programmer: <pvn@ncsa.uiuc.edu> + * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu> * January, 6, 2004 * *------------------------------------------------------------------------- @@ -58,7 +58,7 @@ int main (void) * it returns RET==0 if the objects have the same data *------------------------------------------------------------------------- */ - + /*------------------------------------------------------------------------- * file with all kinds of dataset datatypes *------------------------------------------------------------------------- @@ -178,7 +178,7 @@ int main (void) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("GZIP=9",&pack_options)<0) + if (h5repack_addfilter("GZIP=1",&pack_options)<0) TEST_ERROR; if (h5repack_addlayout("CHUNK=20x10",&pack_options)<0) TEST_ERROR; @@ -212,7 +212,7 @@ int main (void) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("dset2:SZIP=8",&pack_options)<0) + if (h5repack_addfilter("dset2:SZIP=8,EC",&pack_options)<0) TEST_ERROR; if (h5repack_addlayout("dset2:CHUNK=20x10",&pack_options)<0) TEST_ERROR; @@ -240,7 +240,7 @@ TESTING(" adding szip filter to all"); #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("SZIP=8",&pack_options)<0) + if (h5repack_addfilter("SZIP=8,NN",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; @@ -399,8 +399,8 @@ TESTING(" addding shuffle filter to all"); TEST_ERROR; #endif -#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) - if (h5repack_addfilter("dset1:SZIP=8",&pack_options)<0) +#if defined (H5_SZIP_CAN_ENCODE) && defined (H5_HAVE_FILTER_SZIP) + if (h5repack_addfilter("dset1:SZIP=8,NN",&pack_options)<0) TEST_ERROR; #endif @@ -409,7 +409,6 @@ TESTING(" addding shuffle filter to all"); TEST_ERROR; #endif - if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) TEST_ERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) @@ -902,21 +901,26 @@ TESTING(" addding shuffle filter to all"); SKIPPED(); #endif - /*------------------------------------------------------------------------- - * test the NONE global option + * file with all filters + * dset_all + * dset_deflate + * dset_szip + * dset_shuffle + * dset_fletcher32 *------------------------------------------------------------------------- */ + - TESTING(" removing all filters"); + TESTING(" filter conversion from deflate to szip"); #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) \ && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("NONE",&pack_options)<0) + if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME11,FNAME11OUT,&pack_options)<0) TEST_ERROR; @@ -931,54 +935,59 @@ TESTING(" addding shuffle filter to all"); #else SKIPPED(); #endif - - TESTING(" filter conversion from deflate to szip"); -/*------------------------------------------------------------------------- - * filter conversion from deflate to szip - *------------------------------------------------------------------------- - */ -#if (defined (H5_SZIP_CAN_ENCODE) && defined (H5_HAVE_FILTER_SZIP)) && defined(H5_HAVE_FILTER_DEFLATE) + + TESTING(" filter conversion from szip to deflate"); + +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) \ + && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) + if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("dset_deflate:SZIP=8",&pack_options)<0) + if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options)<0) TEST_ERROR; - if (h5repack(FNAME8,FNAME8OUT,&pack_options)<0) + if (h5repack(FNAME11,FNAME11OUT,&pack_options)<0) TEST_ERROR; - if (h5diff(FNAME8,FNAME8OUT,NULL,NULL,&diff_options) == 1) + if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) == 1) TEST_ERROR; - if (h5repack_verify(FNAME8OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11OUT,&pack_options)<=0) TEST_ERROR; if (h5repack_end (&pack_options)<0) TEST_ERROR; + PASSED(); #else SKIPPED(); #endif - TESTING(" filter conversion from szip to deflate"); + /*------------------------------------------------------------------------- - * filter conversion from szip to deflate + * test the NONE global option *------------------------------------------------------------------------- */ -#if (defined (H5_SZIP_CAN_ENCODE) && defined (H5_HAVE_FILTER_SZIP)) && defined(H5_HAVE_FILTER_DEFLATE) + + TESTING(" removing all filters"); + +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) \ + && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) + if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options)<0) + if (h5repack_addfilter("NONE",&pack_options)<0) TEST_ERROR; - if (h5repack(FNAME7,FNAME7OUT,&pack_options)<0) + if (h5repack(FNAME11,FNAME11OUT,&pack_options)<0) TEST_ERROR; - if (h5diff(FNAME7,FNAME7OUT,NULL,NULL,&diff_options) == 1) + if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) == 1) TEST_ERROR; - if (h5repack_verify(FNAME7OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11OUT,&pack_options)<=0) TEST_ERROR; if (h5repack_end (&pack_options)<0) TEST_ERROR; + PASSED(); #else SKIPPED(); #endif - - + /*------------------------------------------------------------------------- * end diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c index 249512c..b0293e3 100644 --- a/tools/h5repack/testh5repack_make.c +++ b/tools/h5repack/testh5repack_make.c @@ -586,7 +586,7 @@ int make_fletcher32(hid_t loc_id) /*------------------------------------------------------------------------- - * checksum + * fletcher32 *------------------------------------------------------------------------- */ #if defined (H5_HAVE_FILTER_FLETCHER32) @@ -625,7 +625,7 @@ out: /*------------------------------------------------------------------------- * Function: make_all * - * Purpose: make a dataset with the all filters + * Purpose: make a file with all filters * *------------------------------------------------------------------------- */ @@ -683,6 +683,69 @@ int make_all(hid_t loc_id) goto out; /*------------------------------------------------------------------------- + * fletcher32 + *------------------------------------------------------------------------- + */ +#if defined (H5_HAVE_FILTER_FLETCHER32) + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set the checksum filter */ + if (H5Pset_fletcher32(dcpl)<0) + goto out; + if (make_dset(loc_id,"dset_fletcher32",sid,dcpl,buf)<0) + goto out; +#endif + +/*------------------------------------------------------------------------- + * SZIP + *------------------------------------------------------------------------- + */ + /* Make sure encoding is enabled */ +#if defined (H5_SZIP_CAN_ENCODE) && defined (H5_HAVE_FILTER_SZIP) + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set szip data */ + if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0) + goto out; + if (make_dset(loc_id,"dset_szip",sid,dcpl,buf)<0) + goto out; +#endif + +/*------------------------------------------------------------------------- + * shuffle + *------------------------------------------------------------------------- + */ +#if defined (H5_HAVE_FILTER_SHUFFLE) + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set the shuffle filter */ + if (H5Pset_shuffle(dcpl)<0) + goto out; + if (make_dset(loc_id,"dset_shuffle",sid,dcpl,buf)<0) + goto out; +#endif + +/*------------------------------------------------------------------------- + * GZIP + *------------------------------------------------------------------------- + */ +#if defined (H5_HAVE_FILTER_DEFLATE) + /* remove the filters from the dcpl */ + if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) + goto out; + /* set deflate data */ + if(H5Pset_deflate(dcpl, 1)<0) + goto out; + if (make_dset(loc_id,"dset_deflate",sid,dcpl,buf)<0) + goto out; +#endif + + + +/*------------------------------------------------------------------------- * close space and dcpl *------------------------------------------------------------------------- */ diff --git a/tools/h5repack/testh5repack_util.c b/tools/h5repack/testh5repack_util.c index 0f7253b..f74b414 100644 --- a/tools/h5repack/testh5repack_util.c +++ b/tools/h5repack/testh5repack_util.c @@ -20,7 +20,7 @@ * * Purpose: utility function to create and write a dataset in LOC_ID * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: November 12, 2003 * @@ -60,7 +60,7 @@ int make_dset(hid_t loc_id, * * Purpose: utility function to create and write a dataset in LOC_ID * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: November 12, 2003 * @@ -106,7 +106,7 @@ int write_dset( hid_t loc_id, * * Purpose: utility function to write an attribute in LOC_ID * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: November 12, 2003 * |