diff options
| -rw-r--r-- | MANIFEST | 5 | ||||
| -rwxr-xr-x | configure | 214 | ||||
| -rw-r--r-- | configure.in | 10 | ||||
| -rw-r--r-- | src/H5.c | 105 | ||||
| -rw-r--r-- | src/H5AC.c | 10 | ||||
| -rw-r--r-- | src/H5B.c | 60 | ||||
| -rw-r--r-- | src/H5Bprivate.h | 1 | ||||
| -rw-r--r-- | src/H5D.c | 66 | ||||
| -rw-r--r-- | src/H5Distore.c | 665 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 1 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 39 | ||||
| -rw-r--r-- | src/H5E.c | 1 | ||||
| -rw-r--r-- | src/H5Epublic.h | 1 | ||||
| -rw-r--r-- | src/H5Farray.c | 33 | ||||
| -rw-r--r-- | src/H5Fistore.c | 665 | ||||
| -rw-r--r-- | src/H5Flow.c | 2 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 7 | ||||
| -rw-r--r-- | src/H5Gnode.c | 1 | ||||
| -rw-r--r-- | src/H5MF.c | 86 | ||||
| -rw-r--r-- | src/H5MFprivate.h | 4 | ||||
| -rw-r--r-- | src/H5MM.c | 15 | ||||
| -rw-r--r-- | src/H5MMprivate.h | 2 | ||||
| -rw-r--r-- | src/H5O.c | 2 | ||||
| -rw-r--r-- | src/H5Ocomp.c | 291 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 14 | ||||
| -rw-r--r-- | src/H5P.c | 173 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 5 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 8 | ||||
| -rw-r--r-- | src/H5Ssimp.c | 24 | ||||
| -rw-r--r-- | src/H5T.c | 133 | ||||
| -rw-r--r-- | src/H5Tconv.c | 24 | ||||
| -rw-r--r-- | src/H5Tpkg.h | 17 | ||||
| -rw-r--r-- | src/H5Tprivate.h | 2 | ||||
| -rw-r--r-- | src/H5Tpublic.h | 9 | ||||
| -rw-r--r-- | src/H5Z.c | 501 | ||||
| -rw-r--r-- | src/H5Zprivate.h | 23 | ||||
| -rw-r--r-- | src/H5Zpublic.h | 69 | ||||
| -rw-r--r-- | src/H5config.h.in | 9 | ||||
| -rw-r--r-- | src/H5private.h | 13 | ||||
| -rw-r--r-- | src/Makefile.in | 12 | ||||
| -rw-r--r-- | src/debug.c | 11 | ||||
| -rw-r--r-- | src/h5ls.c | 25 | ||||
| -rw-r--r-- | src/hdf5.h | 2 | ||||
| -rw-r--r-- | test/.distdep | 44 | ||||
| -rw-r--r-- | test/dsets.c | 145 | ||||
| -rw-r--r-- | test/istore.c | 12 |
46 files changed, 2780 insertions, 781 deletions
@@ -42,6 +42,7 @@ ./html/Big.html ./html/CodeReview.html ./html/Coding.html +./html/Compression.html ./html/Datasets.html ./html/Dataspaces.html ./html/Datatypes.html @@ -152,6 +153,7 @@ ./src/H5MMprivate.h ./src/H5MMpublic.h ./src/H5O.c +./src/H5Ocomp.c ./src/H5Ocont.c ./src/H5Odtype.c ./src/H5Oefl.c @@ -177,6 +179,9 @@ ./src/H5Tpublic.h ./src/H5V.c ./src/H5Vprivate.h +./src/H5Z.c +./src/H5Zprivate.h +./src/H5Zpublic.h ./src/H5config.h.in ./src/H5detect.c ./src/H5private.h @@ -1036,10 +1036,56 @@ EOF else echo "$ac_t""no" 1>&6 fi - + echo $ac_n "checking for compress2 in -lz""... $ac_c" 1>&6 +echo "configure:1041: checking for compress2 in -lz" >&5 +ac_lib_var=`echo z'_'compress2 | sed 'y%./+-%__p_%'` +if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then + echo $ac_n "(cached) $ac_c" 1>&6 +else + ac_save_LIBS="$LIBS" +LIBS="-lz $LIBS" +cat > conftest.$ac_ext <<EOF +#line 1049 "configure" +#include "confdefs.h" +/* Override any gcc2 internal prototype to avoid an error. */ +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char compress2(); + +int main() { +compress2() +; return 0; } +EOF +if { (eval echo configure:1060: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then + rm -rf conftest* + eval "ac_cv_lib_$ac_lib_var=yes" +else + echo "configure: failed program was:" >&5 + cat conftest.$ac_ext >&5 + rm -rf conftest* + eval "ac_cv_lib_$ac_lib_var=no" +fi +rm -f conftest* +LIBS="$ac_save_LIBS" + +fi +if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then + echo "$ac_t""yes" 1>&6 + ac_tr_lib=HAVE_LIB`echo z | sed -e 's/[^a-zA-Z0-9_]/_/g' \ + -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'` + cat >> confdefs.h <<EOF +#define $ac_tr_lib 1 +EOF + + LIBS="-lz $LIBS" + +else + echo "$ac_t""no" 1>&6 +fi + echo $ac_n "checking how to run the C preprocessor""... $ac_c" 1>&6 -echo "configure:1043: checking how to run the C preprocessor" >&5 +echo "configure:1089: checking how to run the C preprocessor" >&5 # On Suns, sometimes $CPP names a directory. if test -n "$CPP" && test -d "$CPP"; then CPP= @@ -1054,13 +1100,13 @@ else # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. cat > conftest.$ac_ext <<EOF -#line 1058 "configure" +#line 1104 "configure" #include "confdefs.h" #include <assert.h> Syntax Error EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1064: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1110: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then : @@ -1071,13 +1117,13 @@ else rm -rf conftest* CPP="${CC-cc} -E -traditional-cpp" cat > conftest.$ac_ext <<EOF -#line 1075 "configure" +#line 1121 "configure" #include "confdefs.h" #include <assert.h> Syntax Error EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1081: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1127: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then : @@ -1100,12 +1146,12 @@ fi echo "$ac_t""$CPP" 1>&6 echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6 -echo "configure:1104: checking for ANSI C header files" >&5 +echo "configure:1150: checking for ANSI C header files" >&5 if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1109 "configure" +#line 1155 "configure" #include "confdefs.h" #include <stdlib.h> #include <stdarg.h> @@ -1113,7 +1159,7 @@ else #include <float.h> EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1117: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1163: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then rm -rf conftest* @@ -1130,7 +1176,7 @@ rm -f conftest* if test $ac_cv_header_stdc = yes; then # SunOS 4.x string.h does not declare mem*, contrary to ANSI. cat > conftest.$ac_ext <<EOF -#line 1134 "configure" +#line 1180 "configure" #include "confdefs.h" #include <string.h> EOF @@ -1148,7 +1194,7 @@ fi if test $ac_cv_header_stdc = yes; then # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. cat > conftest.$ac_ext <<EOF -#line 1152 "configure" +#line 1198 "configure" #include "confdefs.h" #include <stdlib.h> EOF @@ -1169,7 +1215,7 @@ if test "$cross_compiling" = yes; then : else cat > conftest.$ac_ext <<EOF -#line 1173 "configure" +#line 1219 "configure" #include "confdefs.h" #include <ctype.h> #define ISLOWER(c) ('a' <= (c) && (c) <= 'z') @@ -1180,7 +1226,7 @@ if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2); exit (0); } EOF -if { (eval echo configure:1184: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1230: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then : else @@ -1203,21 +1249,21 @@ EOF fi -for ac_hdr in unistd.h +for ac_hdr in unistd.h zlib.h do ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'` echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6 -echo "configure:1211: checking for $ac_hdr" >&5 +echo "configure:1257: checking for $ac_hdr" >&5 if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1216 "configure" +#line 1262 "configure" #include "confdefs.h" #include <$ac_hdr> EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1221: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1267: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then rm -rf conftest* @@ -1246,12 +1292,12 @@ done echo $ac_n "checking for off_t""... $ac_c" 1>&6 -echo "configure:1250: checking for off_t" >&5 +echo "configure:1296: checking for off_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_off_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1255 "configure" +#line 1301 "configure" #include "confdefs.h" #include <sys/types.h> #if STDC_HEADERS @@ -1279,12 +1325,12 @@ EOF fi echo $ac_n "checking for size_t""... $ac_c" 1>&6 -echo "configure:1283: checking for size_t" >&5 +echo "configure:1329: checking for size_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_size_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1288 "configure" +#line 1334 "configure" #include "confdefs.h" #include <sys/types.h> #if STDC_HEADERS @@ -1312,14 +1358,14 @@ EOF fi echo $ac_n "checking whether byte ordering is bigendian""... $ac_c" 1>&6 -echo "configure:1316: checking whether byte ordering is bigendian" >&5 +echo "configure:1362: checking whether byte ordering is bigendian" >&5 if eval "test \"`echo '$''{'ac_cv_c_bigendian'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else ac_cv_c_bigendian=unknown # See if sys/param.h defines the BYTE_ORDER macro. cat > conftest.$ac_ext <<EOF -#line 1323 "configure" +#line 1369 "configure" #include "confdefs.h" #include <sys/types.h> #include <sys/param.h> @@ -1330,11 +1376,11 @@ int main() { #endif ; return 0; } EOF -if { (eval echo configure:1334: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:1380: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* # It does; now see whether it defined to BIG_ENDIAN or not. cat > conftest.$ac_ext <<EOF -#line 1338 "configure" +#line 1384 "configure" #include "confdefs.h" #include <sys/types.h> #include <sys/param.h> @@ -1345,7 +1391,7 @@ int main() { #endif ; return 0; } EOF -if { (eval echo configure:1349: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:1395: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_c_bigendian=yes else @@ -1365,7 +1411,7 @@ if test "$cross_compiling" = yes; then { echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; } else cat > conftest.$ac_ext <<EOF -#line 1369 "configure" +#line 1415 "configure" #include "confdefs.h" main () { /* Are we little or big endian? From Harbison&Steele. */ @@ -1378,7 +1424,7 @@ main () { exit (u.c[sizeof (long) - 1] == 1); } EOF -if { (eval echo configure:1382: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1428: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_c_bigendian=no else @@ -1402,7 +1448,7 @@ EOF fi echo $ac_n "checking size of short""... $ac_c" 1>&6 -echo "configure:1406: checking size of short" >&5 +echo "configure:1452: checking size of short" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_short'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1410,7 +1456,7 @@ else ac_cv_sizeof_short=2 else cat > conftest.$ac_ext <<EOF -#line 1414 "configure" +#line 1460 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1421,7 +1467,7 @@ main() exit(0); } EOF -if { (eval echo configure:1425: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1471: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_short=`cat conftestval` else @@ -1441,7 +1487,7 @@ EOF echo $ac_n "checking size of int""... $ac_c" 1>&6 -echo "configure:1445: checking size of int" >&5 +echo "configure:1491: checking size of int" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_int'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1449,7 +1495,7 @@ else ac_cv_sizeof_int=4 else cat > conftest.$ac_ext <<EOF -#line 1453 "configure" +#line 1499 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1460,7 +1506,7 @@ main() exit(0); } EOF -if { (eval echo configure:1464: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1510: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_int=`cat conftestval` else @@ -1480,7 +1526,7 @@ EOF echo $ac_n "checking size of long""... $ac_c" 1>&6 -echo "configure:1484: checking size of long" >&5 +echo "configure:1530: checking size of long" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_long'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1488,7 +1534,7 @@ else ac_cv_sizeof_long=4 else cat > conftest.$ac_ext <<EOF -#line 1492 "configure" +#line 1538 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1499,7 +1545,7 @@ main() exit(0); } EOF -if { (eval echo configure:1503: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1549: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_long=`cat conftestval` else @@ -1519,7 +1565,7 @@ EOF echo $ac_n "checking size of long long""... $ac_c" 1>&6 -echo "configure:1523: checking size of long long" >&5 +echo "configure:1569: checking size of long long" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_long_long'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1527,7 +1573,7 @@ else ac_cv_sizeof_long_long=8 else cat > conftest.$ac_ext <<EOF -#line 1531 "configure" +#line 1577 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1538,7 +1584,7 @@ main() exit(0); } EOF -if { (eval echo configure:1542: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1588: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_long_long=`cat conftestval` else @@ -1558,7 +1604,7 @@ EOF echo $ac_n "checking size of float""... $ac_c" 1>&6 -echo "configure:1562: checking size of float" >&5 +echo "configure:1608: checking size of float" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_float'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1566,7 +1612,7 @@ else ac_cv_sizeof_float=4 else cat > conftest.$ac_ext <<EOF -#line 1570 "configure" +#line 1616 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1577,7 +1623,7 @@ main() exit(0); } EOF -if { (eval echo configure:1581: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1627: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_float=`cat conftestval` else @@ -1597,7 +1643,7 @@ EOF echo $ac_n "checking size of double""... $ac_c" 1>&6 -echo "configure:1601: checking size of double" >&5 +echo "configure:1647: checking size of double" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_double'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1605,7 +1651,7 @@ else ac_cv_sizeof_double=8 else cat > conftest.$ac_ext <<EOF -#line 1609 "configure" +#line 1655 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1616,7 +1662,7 @@ main() exit(0); } EOF -if { (eval echo configure:1620: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1666: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_double=`cat conftestval` else @@ -1636,7 +1682,7 @@ EOF echo $ac_n "checking size of size_t""... $ac_c" 1>&6 -echo "configure:1640: checking size of size_t" >&5 +echo "configure:1686: checking size of size_t" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_size_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1644,7 +1690,7 @@ else ac_cv_sizeof_size_t=4 else cat > conftest.$ac_ext <<EOF -#line 1648 "configure" +#line 1694 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1655,7 +1701,7 @@ main() exit(0); } EOF -if { (eval echo configure:1659: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1705: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_size_t=`cat conftestval` else @@ -1678,7 +1724,7 @@ cat >>confdefs.h <<\EOF #include <sys/types.h> /*for off_t definition*/ EOF echo $ac_n "checking size of off_t""... $ac_c" 1>&6 -echo "configure:1682: checking size of off_t" >&5 +echo "configure:1728: checking size of off_t" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_off_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1686,7 +1732,7 @@ else ac_cv_sizeof_off_t=4 else cat > conftest.$ac_ext <<EOF -#line 1690 "configure" +#line 1736 "configure" #include "confdefs.h" #include <stdio.h> main() @@ -1697,7 +1743,7 @@ main() exit(0); } EOF -if { (eval echo configure:1701: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1747: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_off_t=`cat conftestval` else @@ -1724,7 +1770,7 @@ if test "${enable_hsizet+set}" = set; then fi echo $ac_n "checking for sizeof hsize_t and hssize_t""... $ac_c" 1>&6 -echo "configure:1728: checking for sizeof hsize_t and hssize_t" >&5 +echo "configure:1774: checking for sizeof hsize_t and hssize_t" >&5 case $HSIZET in no|small) echo "$ac_t""small" 1>&6 @@ -1739,15 +1785,15 @@ EOF esac -for ac_func in getpwuid gethostname system +for ac_func in getpwuid gethostname system getrusage do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 -echo "configure:1746: checking for $ac_func" >&5 +echo "configure:1792: checking for $ac_func" >&5 if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1751 "configure" +#line 1797 "configure" #include "confdefs.h" /* System header to define __stub macros and hopefully few prototypes, which can conflict with char $ac_func(); below. */ @@ -1770,7 +1816,7 @@ $ac_func(); ; return 0; } EOF -if { (eval echo configure:1774: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:1820: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_$ac_func=yes" else @@ -1796,24 +1842,24 @@ done cat > conftest.$ac_ext <<EOF -#line 1800 "configure" +#line 1846 "configure" #include "confdefs.h" #include<sys/types.h> int main() { off64_t n = 0; ; return 0; } EOF -if { (eval echo configure:1807: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:1853: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* for ac_func in lseek64 fseek64 do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 -echo "configure:1812: checking for $ac_func" >&5 +echo "configure:1858: checking for $ac_func" >&5 if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1817 "configure" +#line 1863 "configure" #include "confdefs.h" /* System header to define __stub macros and hopefully few prototypes, which can conflict with char $ac_func(); below. */ @@ -1836,7 +1882,7 @@ $ac_func(); ; return 0; } EOF -if { (eval echo configure:1840: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:1886: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_$ac_func=yes" else @@ -1871,12 +1917,12 @@ rm -f conftest* echo $ac_n "checking for working const""... $ac_c" 1>&6 -echo "configure:1875: checking for working const" >&5 +echo "configure:1921: checking for working const" >&5 if eval "test \"`echo '$''{'ac_cv_c_const'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <<EOF -#line 1880 "configure" +#line 1926 "configure" #include "confdefs.h" int main() { @@ -1925,7 +1971,7 @@ ccp = (char const *const *) p; ; return 0; } EOF -if { (eval echo configure:1929: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:1975: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_c_const=yes else @@ -1946,21 +1992,21 @@ EOF fi echo $ac_n "checking for inline""... $ac_c" 1>&6 -echo "configure:1950: checking for inline" >&5 +echo "configure:1996: checking for inline" >&5 if eval "test \"`echo '$''{'ac_cv_c_inline'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else ac_cv_c_inline=no for ac_kw in inline __inline__ __inline; do cat > conftest.$ac_ext <<EOF -#line 1957 "configure" +#line 2003 "configure" #include "confdefs.h" int main() { } $ac_kw foo() { ; return 0; } EOF -if { (eval echo configure:1964: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2010: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_c_inline=$ac_kw; break else @@ -1987,16 +2033,16 @@ esac echo $ac_n "checking for __attribute__ extension""... $ac_c" 1>&6 -echo "configure:1991: checking for __attribute__ extension" >&5 +echo "configure:2037: checking for __attribute__ extension" >&5 cat > conftest.$ac_ext <<EOF -#line 1993 "configure" +#line 2039 "configure" #include "confdefs.h" int main() { int __attribute__((unused)) f(void){return 1;} ; return 0; } EOF -if { (eval echo configure:2000: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2046: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* cat >> confdefs.h <<\EOF #define HAVE_ATTRIBUTE 1 @@ -2012,16 +2058,16 @@ fi rm -f conftest* echo $ac_n "checking for __FUNCTION__ extension""... $ac_c" 1>&6 -echo "configure:2016: checking for __FUNCTION__ extension" >&5 +echo "configure:2062: checking for __FUNCTION__ extension" >&5 cat > conftest.$ac_ext <<EOF -#line 2018 "configure" +#line 2064 "configure" #include "confdefs.h" int main() { int f(void){return __FUNCTION__;} ; return 0; } EOF -if { (eval echo configure:2025: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2071: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* cat >> confdefs.h <<\EOF #define HAVE_FUNCTION 1 @@ -2037,7 +2083,7 @@ fi rm -f conftest* echo $ac_n "checking for debug flags""... $ac_c" 1>&6 -echo "configure:2041: checking for debug flags" >&5; +echo "configure:2087: checking for debug flags" >&5; # Check whether --enable-debug or --disable-debug was given. if test "${enable_debug+set}" = set; then enableval="$enable_debug" @@ -2045,9 +2091,9 @@ if test "${enable_debug+set}" = set; then fi -all_packages="ac,b,d,e,f,g,hg,hl,i,mf,mm,o,p,s,t,v" +all_packages="ac,b,d,e,f,g,hg,hl,i,mf,mm,o,p,s,t,v,z" if test X = "X$DEBUG_PKG" -o Xyes = "X$DEBUG_PKG"; then - DEBUG_PKG=ac,b,d,e,f,g,hg,i,mm,p,s,t,v + DEBUG_PKG=ac,b,d,e,f,g,hg,i,mm,p,s,t,v,z CPPFLAGS="$CPPFLAGS -UNDEBUG" echo "$ac_t""default ($DEBUG_PKG)" 1>&6 elif test Xall = "X$DEBUG_PKG"; then @@ -2075,7 +2121,7 @@ if test "${enable_parallel+set}" = set; then fi echo $ac_n "checking for parallel support""... $ac_c" 1>&6 -echo "configure:2079: checking for parallel support" >&5; +echo "configure:2125: checking for parallel support" >&5; @@ -2102,7 +2148,7 @@ EOF CFLAGS="$CFLAGS $MPI_LIB" RUNTEST="$RUNTEST" echo $ac_n "checking for main in -lmpi""... $ac_c" 1>&6 -echo "configure:2106: checking for main in -lmpi" >&5 +echo "configure:2152: checking for main in -lmpi" >&5 ac_lib_var=`echo mpi'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2110,14 +2156,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lmpi $LIBS" cat > conftest.$ac_ext <<EOF -#line 2114 "configure" +#line 2160 "configure" #include "confdefs.h" int main() { main() ; return 0; } EOF -if { (eval echo configure:2121: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2167: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -2144,7 +2190,7 @@ else echo "$ac_t""no" 1>&6 fi echo $ac_n "checking for main in -lmpio""... $ac_c" 1>&6 -echo "configure:2148: checking for main in -lmpio" >&5 +echo "configure:2194: checking for main in -lmpio" >&5 ac_lib_var=`echo mpio'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2152,14 +2198,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lmpio $LIBS" cat > conftest.$ac_ext <<EOF -#line 2156 "configure" +#line 2202 "configure" #include "confdefs.h" int main() { main() ; return 0; } EOF -if { (eval echo configure:2163: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2209: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else diff --git a/configure.in b/configure.in index e5b6bb5..b4f36af 100644 --- a/configure.in +++ b/configure.in @@ -126,13 +126,13 @@ dnl ---------------------------------------------------------------------- dnl Check for libraries. (none required yet) dnl AC_CHECK_LIB(coug, main) dnl ...for ASCI/Red - +AC_CHECK_LIB(z, compress2) dnl ---------------------------------------------------------------------- dnl Check for header files. dnl AC_HEADER_STDC -AC_CHECK_HEADERS(unistd.h) +AC_CHECK_HEADERS(unistd.h zlib.h) dnl ---------------------------------------------------------------------- @@ -176,7 +176,7 @@ esac dnl ---------------------------------------------------------------------- dnl Check for functions. dnl -AC_CHECK_FUNCS(getpwuid gethostname system) +AC_CHECK_FUNCS(getpwuid gethostname system getrusage) AC_TRY_COMPILE([#include<sys/types.h>], [off64_t n = 0;], @@ -214,9 +214,9 @@ AC_ARG_ENABLE(debug, is most packages.], DEBUG_PKG=$enableval) -all_packages="ac,b,d,e,f,g,hg,hl,i,mf,mm,o,p,s,t,v" +all_packages="ac,b,d,e,f,g,hg,hl,i,mf,mm,o,p,s,t,v,z" if test X = "X$DEBUG_PKG" -o Xyes = "X$DEBUG_PKG"; then - DEBUG_PKG=ac,b,d,e,f,g,hg,i,mm,p,s,t,v + DEBUG_PKG=ac,b,d,e,f,g,hg,i,mm,p,s,t,v,z CPPFLAGS="$CPPFLAGS -UNDEBUG" AC_MSG_RESULT(default ($DEBUG_PKG)) elif test Xall = "X$DEBUG_PKG"; then @@ -38,6 +38,8 @@ static char RcsId[] = "@(#)$Revision$"; #include <ctype.h> #include <errno.h> #include <stdarg.h> +#include <sys/time.h> +#include <sys/resource.h> /* private headers */ #include <H5private.h> /*library */ @@ -478,7 +480,7 @@ HDfprintf (FILE *stream, const char *fmt, ...) if ('.'==*s) { s++; if (isdigit (*s)) { - prec = (int)strtol (s+1, &rest, 10); + prec = (int)strtol (s, &rest, 10); s = rest; } else if ('*'==*s) { prec = va_arg (ap, int); @@ -576,6 +578,7 @@ HDfprintf (FILE *stream, const char *fmt, ...) long double x = va_arg (ap, long double); n = fprintf (stream, template, x); } + break; case 'a': if (1) { @@ -738,3 +741,103 @@ HDstrtoll (const char *s, const char **rest, int base) if (rest) *rest = s; return acc; } + + +/*------------------------------------------------------------------------- + * Function: H5_timer_reset + * + * Purpose: Resets the timer struct to zero. Use this to reset a timer + * that's being used as an accumulator for summing times. + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +H5_timer_reset (H5_timer_t *timer) +{ + assert (timer); + HDmemset (timer, 0, sizeof *timer); +} + + +/*------------------------------------------------------------------------- + * Function: H5_timer_begin + * + * Purpose: Initialize a timer to time something. + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +H5_timer_begin (H5_timer_t *timer) +{ +#ifdef HAVE_GETRUSAGE + struct rusage rusage; +#endif + struct timeval etime; + + assert (timer); + +#ifdef HAVE_GETRUSAGE + getrusage (RUSAGE_SELF, &rusage); + timer->utime = rusage.ru_utime.tv_sec + rusage.ru_utime.tv_usec/1e6; + timer->stime = rusage.ru_stime.tv_sec + rusage.ru_stime.tv_usec/1e6; +#else + timer->utime = 0.0; + timer->stime = 0.0; +#endif + + gettimeofday (&etime, NULL); + timer->etime = etime.tv_sec + etime.tv_usec/1e6; +} + + + +/*------------------------------------------------------------------------- + * Function: H5_timer_end + * + * Purpose: This function should be called at the end of a timed region. + * The SUM is an optional pointer which will accumulate times. + * TMS is the same struct that was passed to H5_timer_start(). + * On return, TMS will contain total times for the timed region. + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/) +{ + H5_timer_t now; + + assert (timer); + H5_timer_begin (&now); + + timer->utime = now.utime - timer->utime; + timer->stime = now.stime - timer->stime; + timer->etime = now.etime - timer->etime; + + if (sum) { + sum->utime += timer->utime; + sum->stime += timer->stime; + sum->etime += timer->etime; + } +} + @@ -838,9 +838,11 @@ H5AC_debug(H5F_t *f) FUNC_ENTER(H5AC_debug, FAIL); - fprintf(stderr, "H5AC: cache diagnostics for %s\n", f->name); - fprintf(stderr, " %18s %8s %8s %8s %8s+%-8s\n", - "", "Hits", "Misses", "MissRate", "Inits", "Flushes"); + fprintf(stderr, "H5AC: cache statistics for file %s\n", f->name); + fprintf(stderr, " %-18s %8s %8s %8s %8s+%-8s\n", + "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes"); + fprintf(stderr, " %-18s %8s %8s %8s %8s-%-8s\n", + "-----", "----", "------", "--------", "-----", "-------"); for (i = H5AC_BT_ID; i < H5AC_NTYPES; i++) { @@ -876,7 +878,7 @@ H5AC_debug(H5F_t *f) } else { sprintf(ascii, "%7.2f%%", miss_rate); } - fprintf(stderr, " %18s: %8u %8u %7s %8u%+-9ld\n", s, + fprintf(stderr, " %-18s %8u %8u %7s %8u%+-9ld\n", s, cache->diagnostics[i].nhits, cache->diagnostics[i].nmisses, ascii, @@ -1022,7 +1022,7 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, uint8 *lt_key, hbool_t *lt_key_changed, uint8 *md_key, void *udata, uint8 *rt_key, hbool_t *rt_key_changed, - haddr_t *new_node /*out */ ) + haddr_t *new_node/*out*/) { H5B_t *bt = NULL, *twin = NULL, *tmp_bt = NULL; intn lt = 0, idx = -1, rt, cmp = -1; @@ -1097,10 +1097,9 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, idx = 0; if (type->follow_min) { - if ((my_ins = (type->insert) (f, bt->child + idx, - bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, + if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey, + lt_key_changed, md_key, udata, + bt->key[idx+1].nkey, rt_key_changed, &child_addr/*out*/)) < 0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "unable to insert first leaf node"); @@ -1119,11 +1118,11 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR, "unable to decode key"); } - if ((my_ins = H5B_insert_helper(f, bt->child + idx, type, + if ((my_ins = H5B_insert_helper(f, bt->child+idx, type, bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, - &child_addr /*out */ )) < 0) { + md_key, udata, bt->key[idx+1].nkey, + rt_key_changed, + &child_addr/*out*/))<0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert minimum subtree"); } @@ -1138,11 +1137,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR, "unable to decode key"); } - if ((my_ins = (type->insert) (f, bt->child + idx, - bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, - &child_addr /*out */ )) < 0) { + if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey, + lt_key_changed, md_key, udata, + bt->key[idx+1].nkey, rt_key_changed, + &child_addr/*out*/)) < 0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert minimum leaf node"); } @@ -1176,11 +1174,11 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR, "unable to decode key"); } - if ((my_ins = H5B_insert_helper(f, bt->child + idx, type, + if ((my_ins = H5B_insert_helper(f, bt->child+idx, type, bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, - &child_addr /*out */ )) < 0) { + md_key, udata, bt->key[idx+1].nkey, + rt_key_changed, + &child_addr/*out*/)) < 0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert maximum subtree"); } @@ -1195,11 +1193,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, H5B_INS_ERROR, "unable to decode key"); } - if ((my_ins = (type->insert) (f, bt->child + idx, - bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, - &child_addr /*out */ )) < 0) { + if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey, + lt_key_changed, md_key, udata, + bt->key[idx+1].nkey, rt_key_changed, + &child_addr/*out*/)) < 0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert maximum leaf node"); } @@ -1248,11 +1245,10 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, * Follow a branch out of this node to a leaf node of some other type. */ assert(idx >= 0 && idx < bt->nchildren); - if ((my_ins = (type->insert) (f, bt->child + idx, - bt->key[idx].nkey, lt_key_changed, - md_key, udata, - bt->key[idx + 1].nkey, rt_key_changed, - &child_addr /*out */ )) < 0) { + if ((my_ins = (type->insert) (f, bt->child+idx, bt->key[idx].nkey, + lt_key_changed, md_key, udata, + bt->key[idx+1].nkey, rt_key_changed, + &child_addr/*out*/)) < 0) { HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert leaf node"); } @@ -1531,7 +1527,7 @@ H5B_nodesize(H5F_t *f, const H5B_class_t *type, *------------------------------------------------------------------------- */ herr_t -H5B_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, +H5B_debug(H5F_t *f, const haddr_t *addr, FILE *stream, intn indent, intn fwidth, const H5B_class_t *type, void *udata) { H5B_t *bt = NULL; @@ -1599,6 +1595,12 @@ H5B_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, "Address:"); H5F_addr_print(stream, bt->child + i); fprintf(stream, "\n"); + + H5B_decode_key(f, bt, i); + if (type->debug_key) { + (type->debug_key)(stream, indent+3, MAX (0, fwidth-3), + bt->key[i].nkey, udata); + } } FUNC_LEAVE(SUCCEED); diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index b108db8..40aa65c 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -86,6 +86,7 @@ typedef struct H5B_class_t { herr_t (*list) (H5F_t*, const haddr_t*, void*); /*walk leaf nodes*/ herr_t (*decode) (H5F_t*, struct H5B_t*, uint8*, void*); herr_t (*encode) (H5F_t*, struct H5B_t*, uint8*, void*); + herr_t (*debug_key)(FILE*, intn, intn, const void*, const void*); } H5B_class_t; /* @@ -17,16 +17,17 @@ static char RcsId[] = "@(#)$Revision$"; /* $Id$ */ #include <H5private.h> /* Generic Functions */ -#include <H5Iprivate.h> /* IDs */ +#include <H5Iprivate.h> /* IDs */ #include <H5ACprivate.h> /* Cache */ #include <H5Dprivate.h> /* Dataset functions */ #include <H5Eprivate.h> /* Error handling */ #include <H5Gprivate.h> /* Group headers */ -#include <H5HLprivate.h> /* Name heap */ +#include <H5HLprivate.h> /* Name heap */ #include <H5MFprivate.h> /* File space allocation header */ #include <H5MMprivate.h> /* Memory management */ #include <H5Oprivate.h> /* Object headers */ #include <H5Pprivate.h> /* Property lists */ +#include <H5Zprivate.h> /* Data compression */ #define PABLO_MASK H5D_mask @@ -55,10 +56,17 @@ const H5D_create_t H5D_create_dflt = { 1, 1, 1, 1, 1, 1, 1, 1, /*...are quite useless. Larger chunks..*/ 1, 1, 1, 1, 1, 1, 1, 1, /*...produce fewer, but larger I/O......*/ 1, 1, 1, 1, 1, 1, 1, 1}, /*...requests. */ + + /* External file list */ {H5F_ADDR_UNDEF, /* External file list heap address */ 0, /*...slots allocated */ 0, /*...slots used */ - NULL} /*...slot array */ + NULL}, /*...slot array */ + + /* Compression */ + {H5Z_NONE, /* No compression */ + 0, /*...flags */ + 0, NULL} /*...client data */ }; /* Default dataset transfer property list */ @@ -741,6 +749,11 @@ H5D_create(H5G_t *loc, const char *name, const H5T_t *type, const H5S_t *space, assert (type); assert (space); assert (create_parms); + if (H5Z_NONE!=create_parms->compress.method && + H5D_CHUNKED!=create_parms->layout) { + HGOTO_ERROR (H5E_DATASET, H5E_BADVALUE, NULL, + "compression can only be used with chunked layout"); + } /* Initialize the dataset object */ new_dset = H5MM_xcalloc(1, sizeof(H5D_t)); @@ -829,9 +842,17 @@ H5D_create(H5G_t *loc, const char *name, const H5T_t *type, const H5S_t *space, new_dset->type) < 0 || H5S_modify(&(new_dset->ent), new_dset->space) < 0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, - "can't update type or space header messages"); + "unable to update type or space header messages"); } + /* Update the compression message */ + if (H5Z_NONE!=new_dset->create_parms->compress.method && + H5O_modify (&(new_dset->ent), H5O_COMPRESS, 0, H5O_FLAG_CONSTANT, + &(new_dset->create_parms->compress))<0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL, + "unable to update compression header message"); + } + /* * Initialize storage. We assume that external storage is already * initialized by the caller, or at least will be before I/O is @@ -1111,6 +1132,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, size_t target_size; /*desired buffer size */ size_t request_nelmts; /*requested strip mine */ H5T_bkg_t need_bkg; /*type of background buf*/ +#ifdef H5T_DEBUG + H5_timer_t timer; +#endif #ifdef HAVE_PARALLEL int access_mode_saved = -1; #endif @@ -1210,6 +1234,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5T_conv_noop==tconv_func && NULL!=sconv_func->read) { status = (sconv_func->read)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->compress), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf/*out*/); @@ -1296,6 +1321,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, * if necessary. */ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->compress), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, smine_start, smine_nelmts, @@ -1314,14 +1340,19 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, /* * Perform data type conversion. */ +#ifdef H5T_DEBUG + H5T_timer_begin (&timer, cdata); +#endif cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf, - bkg_buf)<0) { + status = (tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf); +#ifdef H5T_DEBUG + H5T_timer_end (&timer, cdata, smine_nelmts); +#endif + if (status<0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed"); } - cdata->nelmts += smine_nelmts; /* * Scatter the data into memory. @@ -1396,6 +1427,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, size_t target_size; /*desired buffer size */ size_t request_nelmts; /*requested strip mine */ H5T_bkg_t need_bkg; /*type of background buf*/ +#ifdef H5T_DEBUG + H5_timer_t timer; +#endif #ifdef HAVE_PARALLEL int access_mode_saved = -1; #endif @@ -1495,6 +1529,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5T_conv_noop==tconv_func && NULL!=sconv_func->write) { status = (sconv_func->write)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->compress), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf); @@ -1588,6 +1623,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, if ((H5D_OPTIMIZE_PIPE && H5T_BKG_YES==need_bkg) || (!H5D_OPTIMIZE_PIPE && need_bkg)) { if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->compress), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, smine_start, smine_nelmts, @@ -1600,19 +1636,25 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, /* * Perform data type conversion. */ +#ifdef H5T_DEBUG + H5T_timer_begin (&timer, cdata); +#endif cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf, - bkg_buf)<0) { + status = (tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf); +#ifdef H5T_DEBUG + H5T_timer_end (&timer, cdata, smine_nelmts); +#endif + if (status<0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed"); } - cdata->nelmts += smine_nelmts; /* * Scatter the data out to the file. */ if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->compress), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, smine_start, smine_nelmts, diff --git a/src/H5Distore.c b/src/H5Distore.c index 1f84aab..7e80f1a 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -14,17 +14,9 @@ #include <H5Oprivate.h> #include <H5Vprivate.h> -typedef enum H5F_isop_t { - H5F_ISTORE_READ, /*read from file to memory */ - H5F_ISTORE_WRITE /*write from memory to file */ -} H5F_isop_t; - -/* Does the array domain include negative indices? */ -#undef H5F_ISTORE_NEGATIVE_DOMAIN - -#define PABLO_MASK H5F_istore_mask /* Interface initialization */ +#define PABLO_MASK H5F_istore_mask static hbool_t interface_initialize_g = FALSE; #define INTERFACE_INIT NULL @@ -48,13 +40,8 @@ static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8 *raw, void *_key); static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8 *raw, void *_key); -static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, - H5F_isop_t op, - const hssize_t offset_f[], - const hsize_t size[], - const hssize_t offset_m[], - const hsize_t size_m[], - void *buf); +static herr_t H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth, + const void *key, const void *udata); /* * B-tree key. A key contains the minimum logical N-dimensional address and @@ -68,12 +55,11 @@ static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, * Only the first few values of the OFFSET and SIZE fields are actually * stored on disk, depending on the dimensionality. * - * The storage file address is part of the B-tree and not part of the key. + * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5F_istore_key_t { - uintn file_number; /*external file number */ + hsize_t nbytes; /*size of stored data */ hssize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ - hsize_t size[H5O_LAYOUT_NDIMS]; /*logical chunk size */ } H5F_istore_key_t; typedef struct H5F_istore_ud1_t { @@ -97,6 +83,7 @@ H5B_class_t H5B_ISTORE[1] = {{ NULL, /*list */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ + H5F_istore_debug_key, /*debug */ }}; @@ -128,9 +115,8 @@ H5F_istore_sizeof_rkey(H5F_t __unused__ *f, const void *_udata) assert(udata); assert(udata->mesg.ndims > 0 && udata->mesg.ndims <= H5O_LAYOUT_NDIMS); - nbytes = 4 + /*external file number */ - udata->mesg.ndims * 4 + /*dimension indices */ - udata->mesg.ndims * 4; /*dimension sizes */ + nbytes = 4 + /*storage size */ + udata->mesg.ndims * 4; /*dimension indices */ return nbytes; } @@ -157,7 +143,7 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) { H5F_istore_key_t *key = (H5F_istore_key_t *) _key; intn i; - intn ndims = (intn)(bt->sizeof_rkey/8); + intn ndims = (intn)((bt->sizeof_rkey-4)/4); FUNC_ENTER(H5F_istore_decode_key, FAIL); @@ -169,11 +155,9 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS); /* decode */ - UINT32DECODE(raw, key->file_number); - assert(0 == key->file_number); + UINT32DECODE (raw, key->nbytes); for (i = 0; i < ndims; i++) { UINT32DECODE(raw, key->offset[i]); - UINT32DECODE(raw, key->size[i]); } FUNC_LEAVE(SUCCEED); @@ -200,7 +184,7 @@ static herr_t H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) { H5F_istore_key_t *key = (H5F_istore_key_t *) _key; - intn ndims = (intn)(bt->sizeof_rkey / 8); + intn ndims = (intn)((bt->sizeof_rkey-4) / 4); intn i; FUNC_ENTER(H5F_istore_encode_key, FAIL); @@ -213,11 +197,9 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS); /* encode */ - UINT32ENCODE(raw, key->file_number); - assert(0 == key->file_number); + UINT32ENCODE (raw, key->nbytes); for (i = 0; i < ndims; i++) { UINT32ENCODE(raw, key->offset[i]); - UINT32ENCODE(raw, key->size[i]); } FUNC_LEAVE(SUCCEED); @@ -225,6 +207,46 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) /*------------------------------------------------------------------------- + * Function: H5F_istore_debug_key + * + * Purpose: Prints a key. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth, + const void *_key, const void *_udata) +{ + const H5F_istore_key_t *key = (const H5F_istore_key_t *)_key; + const H5F_istore_ud1_t *udata = (const H5F_istore_ud1_t *)_udata; + int i; + + FUNC_ENTER (H5F_istore_debug_key, FAIL); + assert (key); + + HDfprintf (stream, "%*s%-*s %Hd bytes\n", indent, "", fwidth, + "Chunk size:", key->nbytes); + HDfprintf (stream, "%*s%-*s {", indent, "", fwidth, + "Logical offset:"); + for (i=0; i<udata->mesg.ndims; i++) { + HDfprintf (stream, "%s%Hd", i?", ":"", key->offset[i]); + } + fputs ("}\n", stream); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_cmp2 * * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer @@ -352,7 +374,6 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op, H5F_istore_key_t *lt_key = (H5F_istore_key_t *) _lt_key; H5F_istore_key_t *rt_key = (H5F_istore_key_t *) _rt_key; H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata; - hsize_t nbytes; intn i; FUNC_ENTER(H5F_istore_new_node, FAIL); @@ -366,37 +387,34 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op, assert(addr); /* Allocate new storage */ - nbytes = H5V_vector_reduce_product(udata->mesg.ndims, udata->key.size); - assert(nbytes > 0); - if (H5MF_alloc(f, H5MF_RAW, nbytes, addr /*out */ ) < 0) { + assert (udata->key.nbytes > 0); + if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, addr /*out */ ) < 0) { HRETURN_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "couldn't allocate new file storage"); } udata->addr = *addr; - udata->key.file_number = 0; - lt_key->file_number = udata->key.file_number; - if (H5B_INS_LEFT != op) - rt_key->file_number = 0; - /* Initialize the key(s) */ - for (i = 0; i < udata->mesg.ndims; i++) { - /* - * The left key describes the storage of the UDATA chunk being - * inserted into the tree. - */ - assert(udata->key.size[i] > 0); + /* + * The left key describes the storage of the UDATA chunk being + * inserted into the tree. + */ + lt_key->nbytes = udata->key.nbytes; + for (i=0; i<udata->mesg.ndims; i++) { lt_key->offset[i] = udata->key.offset[i]; - lt_key->size[i] = udata->key.size[i]; + } - /* - * The right key might already be present. If not, then add - * a zero-width chunk. - */ - if (H5B_INS_LEFT != op) { - assert (udata->key.size[i] < MAX_HSSIZET); + /* + * The right key might already be present. If not, then add a zero-width + * chunk. + */ + if (H5B_INS_LEFT != op) { + rt_key->nbytes = 0; + for (i=0; i<udata->mesg.ndims; i++) { + assert (udata->mesg.dim[i] < MAX_HSSIZET); + assert (udata->key.offset[i]+(hssize_t)(udata->mesg.dim[i]) > + udata->key.offset[i]); rt_key->offset[i] = udata->key.offset[i] + - (hssize_t)udata->key.size[i]; - rt_key->size[i] = 0; + (hssize_t)(udata->mesg.dim[i]); } } @@ -445,12 +463,10 @@ H5F_istore_found(H5F_t __unused__ *f, const haddr_t *addr, /* Initialize return values */ udata->addr = *addr; - udata->key.file_number = lt_key->file_number; - assert(0 == lt_key->file_number); + udata->key.nbytes = lt_key->nbytes; + assert (lt_key->nbytes>0); for (i = 0; i < udata->mesg.ndims; i++) { udata->key.offset[i] = lt_key->offset[i]; - udata->key.size[i] = lt_key->size[i]; - assert(lt_key->size[i] > 0); } FUNC_LEAVE(SUCCEED); @@ -500,7 +516,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata; intn i, cmp; H5B_ins_t ret_value = H5B_INS_ERROR; - hsize_t nbytes; FUNC_ENTER(H5F_istore_insert, H5B_INS_ERROR); @@ -523,46 +538,61 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, assert("HDF5 INTERNAL ERROR -- see rpm" && 0); HRETURN_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, H5B_INS_ERROR, "internal error"); - - } else if (H5V_hyper_eq(udata->mesg.ndims, - udata->key.offset, udata->key.size, - lt_key->offset, lt_key->size)) { + + } else if (H5V_vector_eq_s (udata->mesg.ndims, + udata->key.offset, lt_key->offset) && + lt_key->nbytes>0) { /* - * Already exists. Just return the info. + * Already exists. If the new size is not the same as the old size + * then we should reallocate storage. */ +#if 1 + if (lt_key->nbytes != udata->key.nbytes) { + if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr, + udata->key.nbytes, new_node/*out*/)<0) { + HRETURN_ERROR (H5E_STORAGE, H5E_WRITEERROR, H5B_INS_ERROR, + "unable to reallocate chunk storage"); + } + lt_key->nbytes = udata->key.nbytes; + *lt_key_changed = TRUE; + udata->addr = *new_node; + ret_value = H5B_INS_CHANGE; + } else { + udata->addr = *addr; + ret_value = H5B_INS_NOOP; + } +#else + assert (lt_key->nbytes == udata->key.nbytes); + assert (!H5F_addr_defined (&(udata->addr)) || + H5F_addr_eq (&(udata->addr), addr)); udata->addr = *addr; - udata->key.file_number = lt_key->file_number; ret_value = H5B_INS_NOOP; +#endif } else if (H5V_hyper_disjointp(udata->mesg.ndims, - lt_key->offset, lt_key->size, - udata->key.offset, udata->key.size)) { + lt_key->offset, udata->mesg.dim, + udata->key.offset, udata->mesg.dim)) { assert(H5V_hyper_disjointp(udata->mesg.ndims, - rt_key->offset, rt_key->size, - udata->key.offset, udata->key.size)); - + rt_key->offset, udata->mesg.dim, + udata->key.offset, udata->mesg.dim)); /* * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. */ - md_key->file_number = udata->key.file_number; - for (i=0, nbytes=1; i<udata->mesg.ndims; i++) { + md_key->nbytes = udata->key.nbytes; + for (i=0; i<udata->mesg.ndims; i++) { assert(0 == udata->key.offset[i] % udata->mesg.dim[i]); - assert(udata->key.size[i] == udata->mesg.dim[i]); md_key->offset[i] = udata->key.offset[i]; - md_key->size[i] = udata->key.size[i]; - nbytes *= udata->key.size[i]; } /* * Allocate storage for the new chunk */ - if (H5MF_alloc(f, H5MF_RAW, nbytes, new_node /*out */ ) < 0) { + if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, new_node/*out*/)<0) { HRETURN_ERROR(H5E_IO, H5E_CANTINIT, H5B_INS_ERROR, "file allocation failed"); } udata->addr = *new_node; - udata->key.file_number = 0; ret_value = H5B_INS_RIGHT; } else { @@ -576,43 +606,29 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- - * Function: H5F_istore_copy_hyperslab - * - * Purpose: Reads or writes a hyperslab to disk depending on whether OP - * is H5F_ISTORE_READ or H5F_ISTORE_WRITE. The hyperslab - * storage is described with ISTORE and exists in file F. The - * file hyperslab begins at location OFFSET_F[] (an N-dimensional - * point in the domain in terms of elements) in the file and - * OFFSET_M[] in memory pointed to by BUF. Its size is SIZE[] - * elements. The dimensionality of memory is assumed to be the - * same as the file and the total size of the multi-dimensional - * memory buffer is SIZE_M[]. - * - * The slowest varying dimension is always listed first in the - * various offset and size arrays. - * - * A `chunk' is a hyperslab of the disk array which is stored - * contiguously. I/O occurs in units of chunks where the size of - * a chunk is determined by the alignment constraints specified - * in ISTORE. + * Function: H5F_istore_read + * + * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw + * storage array. * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke - * Friday, October 17, 1997 + * Wednesday, October 15, 1997 * * Modifications: * *------------------------------------------------------------------------- */ -static herr_t -H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, - const hssize_t offset_f[], const hsize_t size[], - const hssize_t offset_m[], const hsize_t size_m[], - void *buf/*in or out*/) +herr_t +H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, const hssize_t offset_f[], + const hsize_t size[], void *buf) { + hssize_t offset_m[H5O_LAYOUT_NDIMS]; + hsize_t size_m[H5O_LAYOUT_NDIMS]; intn i, carry; hsize_t idx_cur[H5O_LAYOUT_NDIMS]; hsize_t idx_min[H5O_LAYOUT_NDIMS]; @@ -621,161 +637,156 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; size_t chunk_size; - hsize_t acc; - uint8 *chunk = NULL; + uint8 *chunk=NULL, *compressed=NULL; H5F_istore_ud1_t udata; herr_t status; herr_t ret_value = FAIL; - FUNC_ENTER(H5F_istore_copy_hyperslab, FAIL); + FUNC_ENTER(H5F_istore_read, FAIL); - /* check args */ - assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(H5F_addr_defined(&(layout->addr))); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); - assert(H5F_ISTORE_READ == op || H5F_ISTORE_WRITE == op); - assert(size); - assert(size_m); - assert(buf); + /* Check args */ + assert (f); + assert (layout && H5D_CHUNKED==layout->type); + assert (layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert (H5F_addr_defined(&(layout->addr))); + assert (offset_f); + assert (size); + assert (buf); + + /* + * For now, a hyperslab of the file must be read into an array in + * memory.We do not yet support reading into a hyperslab of memory. + */ + for (i=0; i<layout->ndims; i++) { + offset_m[i] = 0; + size_m[i] = size[i]; + } + #ifndef NDEBUG for (i=0; i<layout->ndims; i++) { - assert(size_m[i] > 0); /*destination must exist */ - /*hyperslab must fit in BUF */ - assert((offset_m?offset_m[i]:0) + size[i] <= size_m[i]); - assert(layout->dim[i] > 0); + assert (offset_f[i]>=0); /*negative offsets not supported*/ + assert (offset_m[i]>=0); /*negative offsets not supported*/ + assert (size[i]<MAX_SIZET); + assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]); + assert(layout->dim[i]>0); } #endif + /* Determine the chunk size and allocate buffers */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc(chunk_size); + if (comp && H5Z_NONE!=comp->method) { + compressed = H5MM_xmalloc (chunk_size); + } + /* - * As a special case of H5F_ISTORE_READ, if the source is aligned on - * a chunk boundary and is the same size as a chunk, and the destination - * is the same size as a chunk, then instead of reading into a temporary - * buffer and then into the destination, we read directly into the - * destination. + * As a special case if the source is aligned on a chunk boundary and is + * the same size as a chunk, and the destination is the same size as a + * chunk, then instead of reading into a temporary buffer and then into + * the destination, we read directly into the destination. */ - if (H5F_ISTORE_READ==op) { - for (i=0, acc=1; i<layout->ndims; i++) { - if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ - if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ - if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ - udata.key.offset[i] = offset_f[i]; - udata.key.size[i] = layout->dim[i]; - acc *= layout->dim[i]; - } - chunk_size = acc; - assert ((hsize_t)chunk_size==acc); - - if (i==layout->ndims) { - udata.mesg = *layout; - H5F_addr_undef (&(udata.addr)); - udata.key.file_number = 0; - status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined (&(udata.addr))) { - assert (0==udata.key.file_number); - if (H5F_block_read (f, &(udata.addr), (hsize_t)chunk_size, - buf)<0) { + for (i=0; i<layout->ndims; i++) { + if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ + if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ + if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ + udata.key.offset[i] = offset_f[i]; + } + if (i==layout->ndims) { + udata.mesg = *layout; + H5F_addr_undef (&(udata.addr)); + status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); + if (status>=0 && H5F_addr_defined (&(udata.addr))) { + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, + compressed)<0) { HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "unable to read raw storage chunk"); } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, buf)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress raw storage chunk"); + } } else { - HDmemset (buf, 0, chunk_size); + assert (udata.key.nbytes==chunk_size); + if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } } - HRETURN (SUCCEED); + } else { + HDmemset (buf, 0, chunk_size); } + HGOTO_DONE (SUCCEED); } /* * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks copying - * each chunk into a temporary buffer, compressing or decompressing, and - * then copying it to it's destination. + * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing + * or copying each chunk into a temporary buffer, and then copying it to + * it's destination. */ for (i=0; i<layout->ndims; i++) { - idx_min[i] = (offset_f?offset_f[i]:0) / layout->dim[i]; - idx_max[i] = ((offset_f?offset_f[i]:0) + size[i]-1) / layout->dim[i]+1; + idx_min[i] = offset_f[i] / layout->dim[i]; + idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1; idx_cur[i] = idx_min[i]; } - /* Allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - /* Initialize non-changing part of udata */ udata.mesg = *layout; /* Loop over all chunks */ while (1) { - /* Read/Write chunk or create it if it doesn't exist */ - udata.mesg.ndims = layout->ndims; - H5F_addr_undef(&(udata.addr)); - udata.key.file_number = 0; - for (i=0; i<layout->ndims; i++) { - /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); - udata.key.size[i] = layout->dim[i]; /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX((offset_f?offset_f[i]:0), - udata.key.offset[i]) - + offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - udata.key.offset[i]; - sub_size[i] = MIN((idx_cur[i] + 1) * layout->dim[i], - (offset_f ? offset_f[i] : 0) + size[i]) - + sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], + offset_f[i]+size[i]) - (udata.key.offset[i] + offset_wrt_chunk[i]); /* Offset into mem buffer */ sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + - (offset_m ? offset_m[i] : 0) - - (offset_f ? offset_f[i] : 0); - } - - if (H5F_ISTORE_WRITE == op) { - status = H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata); - assert(status >= 0); - } else { - status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); + offset_m[i] - offset_f[i]; } - /* - * If the operation is reading from the disk or if we are writing a - * partial chunk then load the chunk from disk. - */ - if (H5F_ISTORE_READ == op || - !H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || - !H5V_vector_eq_u(layout->ndims, sub_size, udata.key.size)) { - if (status>=0 && H5F_addr_defined(&(udata.addr))) { - assert(0==udata.key.file_number); - if (H5F_block_read(f, &(udata.addr), (hsize_t)chunk_size, - chunk) < 0) { + /* Read chunk */ + H5F_addr_undef(&(udata.addr)); + status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); + if (status>=0 && H5F_addr_defined(&(udata.addr))) { + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, + compressed)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, + chunk)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress data"); + } + } else { + assert (udata.key.nbytes == chunk_size); + if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) { HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw storage chunk"); } - } else { - HDmemset(chunk, 0, chunk_size); - } - } - /* Transfer data to/from the chunk */ - if (H5F_ISTORE_WRITE==op) { - H5V_hyper_copy(layout->ndims, sub_size, - udata.key.size, offset_wrt_chunk, chunk, - size_m, sub_offset_m, buf); - assert(0 == udata.key.file_number); - if (H5F_block_write(f, &(udata.addr), (hsize_t)chunk_size, - chunk) < 0) { - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "unable to write raw storage chunk"); } } else { - H5V_hyper_copy(layout->ndims, sub_size, - size_m, sub_offset_m, (void *)buf, - udata.key.size, offset_wrt_chunk, chunk); + HDmemset(chunk, 0, chunk_size); } + + /* Transfer data from the chunk buffer to the application */ + H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m, + (void *)buf, layout->dim, offset_wrt_chunk, chunk); /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { @@ -787,51 +798,13 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, ret_value = SUCCEED; done: - chunk = H5MM_xfree(chunk); + H5MM_xfree(chunk); + H5MM_xfree (compressed); FUNC_LEAVE(ret_value); } /*------------------------------------------------------------------------- - * Function: H5F_istore_read - * - * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw - * storage array. - * - * Return: Success: SUCCEED - * - * Failure: FAIL - * - * Programmer: Robb Matzke - * Wednesday, October 15, 1997 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, - const hssize_t offset[], const hsize_t size[], void *buf) -{ - FUNC_ENTER(H5F_istore_read, FAIL); - - /* Check args */ - assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); - assert(size); - assert(buf); - - if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_READ, - offset, size, H5V_ZERO, size, buf) < 0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "hyperslab output failure"); - } - FUNC_LEAVE(SUCCEED); -} - - -/*------------------------------------------------------------------------- * Function: H5F_istore_write * * Purpose: Writes a multi-dimensional buffer to (part of) an indexed raw @@ -850,25 +823,174 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, */ herr_t H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, - const hssize_t offset[], const hsize_t size[], - const void *buf) + const H5O_compress_t *comp, const hssize_t offset_f[], + const hsize_t size[], const void *buf) { + hssize_t offset_m[H5O_LAYOUT_NDIMS]; + hsize_t size_m[H5O_LAYOUT_NDIMS]; + intn i, carry; + hsize_t idx_cur[H5O_LAYOUT_NDIMS]; + hsize_t idx_min[H5O_LAYOUT_NDIMS]; + hsize_t idx_max[H5O_LAYOUT_NDIMS]; + hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; + hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; + hsize_t chunk_size, nbytes; + uint8 *chunk=NULL, *compressed=NULL, *outbuf; + H5F_istore_ud1_t udata; + herr_t ret_value = FAIL; + FUNC_ENTER(H5F_istore_write, FAIL); /* Check args */ assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(&(layout->addr))); + assert (offset_f); assert(size); assert(buf); - if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_WRITE, - offset, size, H5V_ZERO, size, - (void*)buf) < 0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "hyperslab output failure"); + /* + * For now the source must not be a hyperslab. It must be an entire + * memory buffer. + */ + for (i=0; i<layout->ndims; i++) { + offset_m[i] = 0; + size_m[i] = size[i]; + } + +#ifndef NDEBUG + for (i=0; i<layout->ndims; i++) { + assert (offset_f[i]>=0); /*negative offsets not supported*/ + assert (offset_m[i]>=0); /*negative offsets not supported*/ + assert(size[i]<MAX_SIZET); + assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]); + assert(layout->dim[i]>0); } - FUNC_LEAVE(SUCCEED); +#endif + + /* + * This is the general case. We set up multi-dimensional counters + * (idx_min, idx_max, and idx_cur) and loop through the chunks copying + * each chunk into a temporary buffer, compressing or decompressing, and + * then copying it to it's destination. + */ + for (i=0; i<layout->ndims; i++) { + idx_min[i] = offset_f[i] / layout->dim[i]; + idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1; + idx_cur[i] = idx_min[i]; + } + + /* Allocate buffers */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc(chunk_size); + if (comp && H5Z_NONE!=comp->method) { + compressed = H5MM_xmalloc (chunk_size); + } + + /* Initialize non-changing part of udata */ + udata.mesg = *layout; + + /* Loop over all chunks */ + while (1) { + + for (i=0; i<layout->ndims; i++) { + /* The location and size of the chunk being accessed */ + assert (layout->dim[i] < MAX_HSSIZET); + udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + + /* The offset and size wrt the chunk */ + offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - + udata.key.offset[i]; + sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], + offset_f[i]+size[i]) - + (udata.key.offset[i] + offset_wrt_chunk[i]); + + /* Offset into mem buffer */ + sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + offset_m[i] - offset_f[i]; + } + + /* + * If we are writing a partial chunk then load the chunk from disk + * and uncompress it if it exists. + */ + if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || + !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) { + if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 && + H5F_addr_defined (&(udata.addr))) { + + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, + compressed)<0) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, + chunk)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress data"); + } + } else { + assert (chunk_size==udata.key.nbytes); + if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, + chunk)<0) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + } + } else { + HDmemset(chunk, 0, chunk_size); + } + } + + /* Transfer data to the chunk */ + H5V_hyper_copy(layout->ndims, sub_size, + layout->dim, offset_wrt_chunk, chunk, + size_m, sub_offset_m, buf); + + /* Compress the chunk */ + if (compressed && + (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) && + nbytes<chunk_size) { + outbuf = compressed; + } else { + outbuf = chunk; + nbytes = chunk_size; + } + + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk + * if its size changed. Then write the data into the file. + */ + H5F_addr_undef(&(udata.addr)); + udata.key.nbytes = nbytes; + if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to allocate chunk"); + } + if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) { + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "unable to write raw storage chunk"); + } + + /* Increment indices */ + for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { + if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i]; + else carry = 0; + } + if (carry) break; + } + ret_value = SUCCEED; + + done: + H5MM_xfree(chunk); + H5MM_xfree (compressed); + FUNC_LEAVE(ret_value); } @@ -921,3 +1043,36 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) FUNC_LEAVE(SUCCEED); } + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_debug + * + * Purpose: Debugs a B-tree node for indexed raw data storage. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, + intn fwidth, int ndims) +{ + H5F_istore_ud1_t udata; + + FUNC_ENTER (H5F_istore_debug, FAIL); + + HDmemset (&udata, 0, sizeof udata); + udata.mesg.ndims = ndims; + + H5B_debug (f, addr, stream, indent, fwidth, H5B_ISTORE, &udata); + + FUNC_LEAVE (SUCCEED); +} diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index f39e5de..e437ce7 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -46,6 +46,7 @@ typedef struct H5D_create_t { intn chunk_ndims; /*chunk dimensionality */ hsize_t chunk_size[32]; /*chunk size if chunked storage */ H5O_efl_t efl; /*external file list */ + H5O_compress_t compress; /*data compression parameters */ } H5D_create_t; /* Dataset transfer property list */ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 634b5b9..d1d5c26 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -1,13 +1,13 @@ /**************************************************************************** - * NCSA HDF * - * Software Development Group * - * National Center for Supercomputing Applications * - * University of Illinois at Urbana-Champaign * - * 605 E. Springfield, Champaign IL 61820 * - * * - * For conditions of distribution and use, see the accompanying * - * hdf/COPYING file. * - * * + * NCSA HDF * + * Software Development Group * + * National Center for Supercomputing Applications * + * University of Illinois at Urbana-Champaign * + * 605 E. Springfield, Champaign IL 61820 * + * * + * For conditions of distribution and use, see the accompanying * + * hdf/COPYING file. * + * * ****************************************************************************/ /* @@ -22,38 +22,39 @@ /* Values for the H5D_LAYOUT property */ typedef enum H5D_layout_t { - H5D_LAYOUT_ERROR = -1, + H5D_LAYOUT_ERROR = -1, - H5D_COMPACT = 0, /*raw data is very small */ - H5D_CONTIGUOUS = 1, /*the default */ - H5D_CHUNKED = 2, /*slow and fancy */ + H5D_COMPACT = 0, /*raw data is very small */ + H5D_CONTIGUOUS = 1, /*the default */ + H5D_CHUNKED = 2, /*slow and fancy */ - H5D_NLAYOUTS = 3 /*this one must be last! */ + H5D_NLAYOUTS = 3 /*this one must be last! */ } H5D_layout_t; #ifdef HAVE_PARALLEL /* Values for the data transfer property */ typedef enum H5D_transfer_t { - H5D_XFER_INDEPENDENT, /*Independent data transfer */ - H5D_XFER_COLLECTIVE /*Collective data transfer */ + H5D_XFER_INDEPENDENT, /*Independent data transfer */ + H5D_XFER_COLLECTIVE /*Collective data transfer */ } H5D_transfer_t; #endif + #ifdef __cplusplus extern "C" { #endif hid_t H5Dcreate (hid_t file_id, const char *name, hid_t type_id, - hid_t space_id, hid_t create_parms_id); + hid_t space_id, hid_t create_parms_id); hid_t H5Dopen (hid_t file_id, const char *name); herr_t H5Dclose (hid_t dataset_id); hid_t H5Dget_space (hid_t dataset_id); hid_t H5Dget_type (hid_t dataset_id); hid_t H5Dget_create_plist (hid_t dataset_id); herr_t H5Dread (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id, - hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/); + hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/); herr_t H5Dwrite (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id, - hid_t file_space_id, hid_t xfer_parms_id, const void *buf); + hid_t file_space_id, hid_t xfer_parms_id, const void *buf); herr_t H5Dextend (hid_t dataset_id, const hsize_t *size); #ifdef __cplusplus @@ -54,6 +54,7 @@ static const H5E_major_mesg_t H5E_major_mesg_g[] = { {H5E_DATASET, "Dataset interface"}, {H5E_STORAGE, "Data storage layer"}, {H5E_TEMPLATE, "Property list interface"}, + {H5E_COMP, "Data compression layer"}, {H5E_EFL, "External file list"}, }; diff --git a/src/H5Epublic.h b/src/H5Epublic.h index ee228e2..284b310 100644 --- a/src/H5Epublic.h +++ b/src/H5Epublic.h @@ -46,6 +46,7 @@ typedef enum H5E_major_t { H5E_DATASET, /*Dataset */ H5E_STORAGE, /*data storage */ H5E_TEMPLATE, /*Property lists */ + H5E_COMP, /*Data compression */ H5E_EFL /*External file list */ } H5E_major_t; diff --git a/src/H5Farray.c b/src/H5Farray.c index 98d3022..1f2bb6c 100644 --- a/src/H5Farray.c +++ b/src/H5Farray.c @@ -112,7 +112,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/) */ herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, const hsize_t _hslab_size[], const hsize_t mem_size[], const hssize_t mem_offset[], const hssize_t file_offset[], void *_buf/*out*/) @@ -173,6 +173,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, "negative offsets are not valid"); } } + + /* + * Compression cannot be used for contiguous data. + */ + if (comp && H5Z_NONE!=comp->method) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "compression is not allowed for contiguous data"); + } /* * Calculate the strides needed to walk through the array on disk @@ -270,7 +278,8 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max); "unable to copy into a proper hyperslab"); } } - if (H5F_istore_read (f, layout, file_offset, hslab_size, buf)<0) { + if (H5F_istore_read (f, layout, comp, file_offset, hslab_size, + buf)<0) { HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, "chunked read failed"); } break; @@ -313,9 +322,10 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max); */ herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, const hsize_t _hslab_size[], - const hsize_t mem_size[], const hssize_t mem_offset[], - const hssize_t file_offset[], const void *_buf) + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, + const hsize_t _hslab_size[], const hsize_t mem_size[], + const hssize_t mem_offset[], const hssize_t file_offset[], + const void *_buf) { const uint8 *buf = (const uint8 *)_buf; /*cast for arithmetic */ hssize_t file_stride[H5O_LAYOUT_NDIMS]; /*strides through file */ @@ -369,10 +379,18 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, */ for (i=0; i<ndims; i++) { if (mem_offset[i]<0 || file_offset[i]<0) { - HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "negative offsets are not valid"); } } + + /* + * Compression cannot be used for contiguous data + */ + if (comp && H5Z_NONE!=comp->method) { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "compression is not allowed for contiguous data"); + } /* * Calculate the strides needed to walk through the array on disk. @@ -471,7 +489,8 @@ printf("nelmts=%lu, min=%lu, max=%lu\n", temp, min, max); "unable to copy from a proper hyperslab"); } } - if (H5F_istore_write (f, layout, file_offset, hslab_size, buf)<0) { + if (H5F_istore_write (f, layout, comp, file_offset, hslab_size, + buf)<0) { HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "chunked write failed"); } diff --git a/src/H5Fistore.c b/src/H5Fistore.c index 1f84aab..7e80f1a 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -14,17 +14,9 @@ #include <H5Oprivate.h> #include <H5Vprivate.h> -typedef enum H5F_isop_t { - H5F_ISTORE_READ, /*read from file to memory */ - H5F_ISTORE_WRITE /*write from memory to file */ -} H5F_isop_t; - -/* Does the array domain include negative indices? */ -#undef H5F_ISTORE_NEGATIVE_DOMAIN - -#define PABLO_MASK H5F_istore_mask /* Interface initialization */ +#define PABLO_MASK H5F_istore_mask static hbool_t interface_initialize_g = FALSE; #define INTERFACE_INIT NULL @@ -48,13 +40,8 @@ static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8 *raw, void *_key); static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8 *raw, void *_key); -static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, - H5F_isop_t op, - const hssize_t offset_f[], - const hsize_t size[], - const hssize_t offset_m[], - const hsize_t size_m[], - void *buf); +static herr_t H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth, + const void *key, const void *udata); /* * B-tree key. A key contains the minimum logical N-dimensional address and @@ -68,12 +55,11 @@ static herr_t H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, * Only the first few values of the OFFSET and SIZE fields are actually * stored on disk, depending on the dimensionality. * - * The storage file address is part of the B-tree and not part of the key. + * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5F_istore_key_t { - uintn file_number; /*external file number */ + hsize_t nbytes; /*size of stored data */ hssize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ - hsize_t size[H5O_LAYOUT_NDIMS]; /*logical chunk size */ } H5F_istore_key_t; typedef struct H5F_istore_ud1_t { @@ -97,6 +83,7 @@ H5B_class_t H5B_ISTORE[1] = {{ NULL, /*list */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ + H5F_istore_debug_key, /*debug */ }}; @@ -128,9 +115,8 @@ H5F_istore_sizeof_rkey(H5F_t __unused__ *f, const void *_udata) assert(udata); assert(udata->mesg.ndims > 0 && udata->mesg.ndims <= H5O_LAYOUT_NDIMS); - nbytes = 4 + /*external file number */ - udata->mesg.ndims * 4 + /*dimension indices */ - udata->mesg.ndims * 4; /*dimension sizes */ + nbytes = 4 + /*storage size */ + udata->mesg.ndims * 4; /*dimension indices */ return nbytes; } @@ -157,7 +143,7 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) { H5F_istore_key_t *key = (H5F_istore_key_t *) _key; intn i; - intn ndims = (intn)(bt->sizeof_rkey/8); + intn ndims = (intn)((bt->sizeof_rkey-4)/4); FUNC_ENTER(H5F_istore_decode_key, FAIL); @@ -169,11 +155,9 @@ H5F_istore_decode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS); /* decode */ - UINT32DECODE(raw, key->file_number); - assert(0 == key->file_number); + UINT32DECODE (raw, key->nbytes); for (i = 0; i < ndims; i++) { UINT32DECODE(raw, key->offset[i]); - UINT32DECODE(raw, key->size[i]); } FUNC_LEAVE(SUCCEED); @@ -200,7 +184,7 @@ static herr_t H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) { H5F_istore_key_t *key = (H5F_istore_key_t *) _key; - intn ndims = (intn)(bt->sizeof_rkey / 8); + intn ndims = (intn)((bt->sizeof_rkey-4) / 4); intn i; FUNC_ENTER(H5F_istore_encode_key, FAIL); @@ -213,11 +197,9 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) assert(ndims > 0 && ndims <= H5O_LAYOUT_NDIMS); /* encode */ - UINT32ENCODE(raw, key->file_number); - assert(0 == key->file_number); + UINT32ENCODE (raw, key->nbytes); for (i = 0; i < ndims; i++) { UINT32ENCODE(raw, key->offset[i]); - UINT32ENCODE(raw, key->size[i]); } FUNC_LEAVE(SUCCEED); @@ -225,6 +207,46 @@ H5F_istore_encode_key(H5F_t __unused__ *f, H5B_t *bt, uint8 *raw, void *_key) /*------------------------------------------------------------------------- + * Function: H5F_istore_debug_key + * + * Purpose: Prints a key. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_debug_key (FILE *stream, intn indent, intn fwidth, + const void *_key, const void *_udata) +{ + const H5F_istore_key_t *key = (const H5F_istore_key_t *)_key; + const H5F_istore_ud1_t *udata = (const H5F_istore_ud1_t *)_udata; + int i; + + FUNC_ENTER (H5F_istore_debug_key, FAIL); + assert (key); + + HDfprintf (stream, "%*s%-*s %Hd bytes\n", indent, "", fwidth, + "Chunk size:", key->nbytes); + HDfprintf (stream, "%*s%-*s {", indent, "", fwidth, + "Logical offset:"); + for (i=0; i<udata->mesg.ndims; i++) { + HDfprintf (stream, "%s%Hd", i?", ":"", key->offset[i]); + } + fputs ("}\n", stream); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_cmp2 * * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer @@ -352,7 +374,6 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op, H5F_istore_key_t *lt_key = (H5F_istore_key_t *) _lt_key; H5F_istore_key_t *rt_key = (H5F_istore_key_t *) _rt_key; H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata; - hsize_t nbytes; intn i; FUNC_ENTER(H5F_istore_new_node, FAIL); @@ -366,37 +387,34 @@ H5F_istore_new_node(H5F_t *f, H5B_ins_t op, assert(addr); /* Allocate new storage */ - nbytes = H5V_vector_reduce_product(udata->mesg.ndims, udata->key.size); - assert(nbytes > 0); - if (H5MF_alloc(f, H5MF_RAW, nbytes, addr /*out */ ) < 0) { + assert (udata->key.nbytes > 0); + if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, addr /*out */ ) < 0) { HRETURN_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "couldn't allocate new file storage"); } udata->addr = *addr; - udata->key.file_number = 0; - lt_key->file_number = udata->key.file_number; - if (H5B_INS_LEFT != op) - rt_key->file_number = 0; - /* Initialize the key(s) */ - for (i = 0; i < udata->mesg.ndims; i++) { - /* - * The left key describes the storage of the UDATA chunk being - * inserted into the tree. - */ - assert(udata->key.size[i] > 0); + /* + * The left key describes the storage of the UDATA chunk being + * inserted into the tree. + */ + lt_key->nbytes = udata->key.nbytes; + for (i=0; i<udata->mesg.ndims; i++) { lt_key->offset[i] = udata->key.offset[i]; - lt_key->size[i] = udata->key.size[i]; + } - /* - * The right key might already be present. If not, then add - * a zero-width chunk. - */ - if (H5B_INS_LEFT != op) { - assert (udata->key.size[i] < MAX_HSSIZET); + /* + * The right key might already be present. If not, then add a zero-width + * chunk. + */ + if (H5B_INS_LEFT != op) { + rt_key->nbytes = 0; + for (i=0; i<udata->mesg.ndims; i++) { + assert (udata->mesg.dim[i] < MAX_HSSIZET); + assert (udata->key.offset[i]+(hssize_t)(udata->mesg.dim[i]) > + udata->key.offset[i]); rt_key->offset[i] = udata->key.offset[i] + - (hssize_t)udata->key.size[i]; - rt_key->size[i] = 0; + (hssize_t)(udata->mesg.dim[i]); } } @@ -445,12 +463,10 @@ H5F_istore_found(H5F_t __unused__ *f, const haddr_t *addr, /* Initialize return values */ udata->addr = *addr; - udata->key.file_number = lt_key->file_number; - assert(0 == lt_key->file_number); + udata->key.nbytes = lt_key->nbytes; + assert (lt_key->nbytes>0); for (i = 0; i < udata->mesg.ndims; i++) { udata->key.offset[i] = lt_key->offset[i]; - udata->key.size[i] = lt_key->size[i]; - assert(lt_key->size[i] > 0); } FUNC_LEAVE(SUCCEED); @@ -500,7 +516,6 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, H5F_istore_ud1_t *udata = (H5F_istore_ud1_t *) _udata; intn i, cmp; H5B_ins_t ret_value = H5B_INS_ERROR; - hsize_t nbytes; FUNC_ENTER(H5F_istore_insert, H5B_INS_ERROR); @@ -523,46 +538,61 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, assert("HDF5 INTERNAL ERROR -- see rpm" && 0); HRETURN_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, H5B_INS_ERROR, "internal error"); - - } else if (H5V_hyper_eq(udata->mesg.ndims, - udata->key.offset, udata->key.size, - lt_key->offset, lt_key->size)) { + + } else if (H5V_vector_eq_s (udata->mesg.ndims, + udata->key.offset, lt_key->offset) && + lt_key->nbytes>0) { /* - * Already exists. Just return the info. + * Already exists. If the new size is not the same as the old size + * then we should reallocate storage. */ +#if 1 + if (lt_key->nbytes != udata->key.nbytes) { + if (H5MF_realloc (f, H5MF_RAW, lt_key->nbytes, addr, + udata->key.nbytes, new_node/*out*/)<0) { + HRETURN_ERROR (H5E_STORAGE, H5E_WRITEERROR, H5B_INS_ERROR, + "unable to reallocate chunk storage"); + } + lt_key->nbytes = udata->key.nbytes; + *lt_key_changed = TRUE; + udata->addr = *new_node; + ret_value = H5B_INS_CHANGE; + } else { + udata->addr = *addr; + ret_value = H5B_INS_NOOP; + } +#else + assert (lt_key->nbytes == udata->key.nbytes); + assert (!H5F_addr_defined (&(udata->addr)) || + H5F_addr_eq (&(udata->addr), addr)); udata->addr = *addr; - udata->key.file_number = lt_key->file_number; ret_value = H5B_INS_NOOP; +#endif } else if (H5V_hyper_disjointp(udata->mesg.ndims, - lt_key->offset, lt_key->size, - udata->key.offset, udata->key.size)) { + lt_key->offset, udata->mesg.dim, + udata->key.offset, udata->mesg.dim)) { assert(H5V_hyper_disjointp(udata->mesg.ndims, - rt_key->offset, rt_key->size, - udata->key.offset, udata->key.size)); - + rt_key->offset, udata->mesg.dim, + udata->key.offset, udata->mesg.dim)); /* * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. */ - md_key->file_number = udata->key.file_number; - for (i=0, nbytes=1; i<udata->mesg.ndims; i++) { + md_key->nbytes = udata->key.nbytes; + for (i=0; i<udata->mesg.ndims; i++) { assert(0 == udata->key.offset[i] % udata->mesg.dim[i]); - assert(udata->key.size[i] == udata->mesg.dim[i]); md_key->offset[i] = udata->key.offset[i]; - md_key->size[i] = udata->key.size[i]; - nbytes *= udata->key.size[i]; } /* * Allocate storage for the new chunk */ - if (H5MF_alloc(f, H5MF_RAW, nbytes, new_node /*out */ ) < 0) { + if (H5MF_alloc(f, H5MF_RAW, udata->key.nbytes, new_node/*out*/)<0) { HRETURN_ERROR(H5E_IO, H5E_CANTINIT, H5B_INS_ERROR, "file allocation failed"); } udata->addr = *new_node; - udata->key.file_number = 0; ret_value = H5B_INS_RIGHT; } else { @@ -576,43 +606,29 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- - * Function: H5F_istore_copy_hyperslab - * - * Purpose: Reads or writes a hyperslab to disk depending on whether OP - * is H5F_ISTORE_READ or H5F_ISTORE_WRITE. The hyperslab - * storage is described with ISTORE and exists in file F. The - * file hyperslab begins at location OFFSET_F[] (an N-dimensional - * point in the domain in terms of elements) in the file and - * OFFSET_M[] in memory pointed to by BUF. Its size is SIZE[] - * elements. The dimensionality of memory is assumed to be the - * same as the file and the total size of the multi-dimensional - * memory buffer is SIZE_M[]. - * - * The slowest varying dimension is always listed first in the - * various offset and size arrays. - * - * A `chunk' is a hyperslab of the disk array which is stored - * contiguously. I/O occurs in units of chunks where the size of - * a chunk is determined by the alignment constraints specified - * in ISTORE. + * Function: H5F_istore_read + * + * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw + * storage array. * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke - * Friday, October 17, 1997 + * Wednesday, October 15, 1997 * * Modifications: * *------------------------------------------------------------------------- */ -static herr_t -H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, - const hssize_t offset_f[], const hsize_t size[], - const hssize_t offset_m[], const hsize_t size_m[], - void *buf/*in or out*/) +herr_t +H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, + const H5O_compress_t *comp, const hssize_t offset_f[], + const hsize_t size[], void *buf) { + hssize_t offset_m[H5O_LAYOUT_NDIMS]; + hsize_t size_m[H5O_LAYOUT_NDIMS]; intn i, carry; hsize_t idx_cur[H5O_LAYOUT_NDIMS]; hsize_t idx_min[H5O_LAYOUT_NDIMS]; @@ -621,161 +637,156 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; size_t chunk_size; - hsize_t acc; - uint8 *chunk = NULL; + uint8 *chunk=NULL, *compressed=NULL; H5F_istore_ud1_t udata; herr_t status; herr_t ret_value = FAIL; - FUNC_ENTER(H5F_istore_copy_hyperslab, FAIL); + FUNC_ENTER(H5F_istore_read, FAIL); - /* check args */ - assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(H5F_addr_defined(&(layout->addr))); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); - assert(H5F_ISTORE_READ == op || H5F_ISTORE_WRITE == op); - assert(size); - assert(size_m); - assert(buf); + /* Check args */ + assert (f); + assert (layout && H5D_CHUNKED==layout->type); + assert (layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert (H5F_addr_defined(&(layout->addr))); + assert (offset_f); + assert (size); + assert (buf); + + /* + * For now, a hyperslab of the file must be read into an array in + * memory.We do not yet support reading into a hyperslab of memory. + */ + for (i=0; i<layout->ndims; i++) { + offset_m[i] = 0; + size_m[i] = size[i]; + } + #ifndef NDEBUG for (i=0; i<layout->ndims; i++) { - assert(size_m[i] > 0); /*destination must exist */ - /*hyperslab must fit in BUF */ - assert((offset_m?offset_m[i]:0) + size[i] <= size_m[i]); - assert(layout->dim[i] > 0); + assert (offset_f[i]>=0); /*negative offsets not supported*/ + assert (offset_m[i]>=0); /*negative offsets not supported*/ + assert (size[i]<MAX_SIZET); + assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]); + assert(layout->dim[i]>0); } #endif + /* Determine the chunk size and allocate buffers */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc(chunk_size); + if (comp && H5Z_NONE!=comp->method) { + compressed = H5MM_xmalloc (chunk_size); + } + /* - * As a special case of H5F_ISTORE_READ, if the source is aligned on - * a chunk boundary and is the same size as a chunk, and the destination - * is the same size as a chunk, then instead of reading into a temporary - * buffer and then into the destination, we read directly into the - * destination. + * As a special case if the source is aligned on a chunk boundary and is + * the same size as a chunk, and the destination is the same size as a + * chunk, then instead of reading into a temporary buffer and then into + * the destination, we read directly into the destination. */ - if (H5F_ISTORE_READ==op) { - for (i=0, acc=1; i<layout->ndims; i++) { - if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ - if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ - if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ - udata.key.offset[i] = offset_f[i]; - udata.key.size[i] = layout->dim[i]; - acc *= layout->dim[i]; - } - chunk_size = acc; - assert ((hsize_t)chunk_size==acc); - - if (i==layout->ndims) { - udata.mesg = *layout; - H5F_addr_undef (&(udata.addr)); - udata.key.file_number = 0; - status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); - if (status>=0 && H5F_addr_defined (&(udata.addr))) { - assert (0==udata.key.file_number); - if (H5F_block_read (f, &(udata.addr), (hsize_t)chunk_size, - buf)<0) { + for (i=0; i<layout->ndims; i++) { + if (offset_f[i] % layout->dim[i]) break; /*src not aligned*/ + if (size[i]!=layout->dim[i]) break; /*src not a chunk*/ + if (size_m[i]!=layout->dim[i]) break; /*dst not a chunk*/ + udata.key.offset[i] = offset_f[i]; + } + if (i==layout->ndims) { + udata.mesg = *layout; + H5F_addr_undef (&(udata.addr)); + status = H5B_find (f, H5B_ISTORE, &(layout->addr), &udata); + if (status>=0 && H5F_addr_defined (&(udata.addr))) { + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, + compressed)<0) { HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "unable to read raw storage chunk"); } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, buf)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress raw storage chunk"); + } } else { - HDmemset (buf, 0, chunk_size); + assert (udata.key.nbytes==chunk_size); + if (H5F_block_read (f, &(udata.addr), chunk_size, buf)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } } - HRETURN (SUCCEED); + } else { + HDmemset (buf, 0, chunk_size); } + HGOTO_DONE (SUCCEED); } /* * This is the general case. We set up multi-dimensional counters - * (idx_min, idx_max, and idx_cur) and loop through the chunks copying - * each chunk into a temporary buffer, compressing or decompressing, and - * then copying it to it's destination. + * (idx_min, idx_max, and idx_cur) and loop through the chunks compressing + * or copying each chunk into a temporary buffer, and then copying it to + * it's destination. */ for (i=0; i<layout->ndims; i++) { - idx_min[i] = (offset_f?offset_f[i]:0) / layout->dim[i]; - idx_max[i] = ((offset_f?offset_f[i]:0) + size[i]-1) / layout->dim[i]+1; + idx_min[i] = offset_f[i] / layout->dim[i]; + idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1; idx_cur[i] = idx_min[i]; } - /* Allocate buffers */ - for (i=0, chunk_size=1; i<layout->ndims; i++) { - chunk_size *= layout->dim[i]; - } - chunk = H5MM_xmalloc(chunk_size); - /* Initialize non-changing part of udata */ udata.mesg = *layout; /* Loop over all chunks */ while (1) { - /* Read/Write chunk or create it if it doesn't exist */ - udata.mesg.ndims = layout->ndims; - H5F_addr_undef(&(udata.addr)); - udata.key.file_number = 0; - for (i=0; i<layout->ndims; i++) { - /* The location and size of the chunk being accessed */ assert (layout->dim[i] < MAX_HSSIZET); udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); - udata.key.size[i] = layout->dim[i]; /* The offset and size wrt the chunk */ - offset_wrt_chunk[i] = MAX((offset_f?offset_f[i]:0), - udata.key.offset[i]) - + offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - udata.key.offset[i]; - sub_size[i] = MIN((idx_cur[i] + 1) * layout->dim[i], - (offset_f ? offset_f[i] : 0) + size[i]) - + sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], + offset_f[i]+size[i]) - (udata.key.offset[i] + offset_wrt_chunk[i]); /* Offset into mem buffer */ sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + - (offset_m ? offset_m[i] : 0) - - (offset_f ? offset_f[i] : 0); - } - - if (H5F_ISTORE_WRITE == op) { - status = H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata); - assert(status >= 0); - } else { - status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); + offset_m[i] - offset_f[i]; } - /* - * If the operation is reading from the disk or if we are writing a - * partial chunk then load the chunk from disk. - */ - if (H5F_ISTORE_READ == op || - !H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || - !H5V_vector_eq_u(layout->ndims, sub_size, udata.key.size)) { - if (status>=0 && H5F_addr_defined(&(udata.addr))) { - assert(0==udata.key.file_number); - if (H5F_block_read(f, &(udata.addr), (hsize_t)chunk_size, - chunk) < 0) { + /* Read chunk */ + H5F_addr_undef(&(udata.addr)); + status = H5B_find(f, H5B_ISTORE, &(layout->addr), &udata); + if (status>=0 && H5F_addr_defined(&(udata.addr))) { + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read (f, &(udata.addr), udata.key.nbytes, + compressed)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, + chunk)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress data"); + } + } else { + assert (udata.key.nbytes == chunk_size); + if (H5F_block_read(f, &(udata.addr), chunk_size, chunk) < 0) { HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw storage chunk"); } - } else { - HDmemset(chunk, 0, chunk_size); - } - } - /* Transfer data to/from the chunk */ - if (H5F_ISTORE_WRITE==op) { - H5V_hyper_copy(layout->ndims, sub_size, - udata.key.size, offset_wrt_chunk, chunk, - size_m, sub_offset_m, buf); - assert(0 == udata.key.file_number); - if (H5F_block_write(f, &(udata.addr), (hsize_t)chunk_size, - chunk) < 0) { - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "unable to write raw storage chunk"); } } else { - H5V_hyper_copy(layout->ndims, sub_size, - size_m, sub_offset_m, (void *)buf, - udata.key.size, offset_wrt_chunk, chunk); + HDmemset(chunk, 0, chunk_size); } + + /* Transfer data from the chunk buffer to the application */ + H5V_hyper_copy(layout->ndims, sub_size, size_m, sub_offset_m, + (void *)buf, layout->dim, offset_wrt_chunk, chunk); /* Increment indices */ for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { @@ -787,51 +798,13 @@ H5F_istore_copy_hyperslab(H5F_t *f, const H5O_layout_t *layout, H5F_isop_t op, ret_value = SUCCEED; done: - chunk = H5MM_xfree(chunk); + H5MM_xfree(chunk); + H5MM_xfree (compressed); FUNC_LEAVE(ret_value); } /*------------------------------------------------------------------------- - * Function: H5F_istore_read - * - * Purpose: Reads a multi-dimensional buffer from (part of) an indexed raw - * storage array. - * - * Return: Success: SUCCEED - * - * Failure: FAIL - * - * Programmer: Robb Matzke - * Wednesday, October 15, 1997 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, - const hssize_t offset[], const hsize_t size[], void *buf) -{ - FUNC_ENTER(H5F_istore_read, FAIL); - - /* Check args */ - assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); - assert(size); - assert(buf); - - if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_READ, - offset, size, H5V_ZERO, size, buf) < 0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "hyperslab output failure"); - } - FUNC_LEAVE(SUCCEED); -} - - -/*------------------------------------------------------------------------- * Function: H5F_istore_write * * Purpose: Writes a multi-dimensional buffer to (part of) an indexed raw @@ -850,25 +823,174 @@ H5F_istore_read(H5F_t *f, const H5O_layout_t *layout, */ herr_t H5F_istore_write(H5F_t *f, const H5O_layout_t *layout, - const hssize_t offset[], const hsize_t size[], - const void *buf) + const H5O_compress_t *comp, const hssize_t offset_f[], + const hsize_t size[], const void *buf) { + hssize_t offset_m[H5O_LAYOUT_NDIMS]; + hsize_t size_m[H5O_LAYOUT_NDIMS]; + intn i, carry; + hsize_t idx_cur[H5O_LAYOUT_NDIMS]; + hsize_t idx_min[H5O_LAYOUT_NDIMS]; + hsize_t idx_max[H5O_LAYOUT_NDIMS]; + hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hssize_t offset_wrt_chunk[H5O_LAYOUT_NDIMS]; + hssize_t sub_offset_m[H5O_LAYOUT_NDIMS]; + hsize_t chunk_size, nbytes; + uint8 *chunk=NULL, *compressed=NULL, *outbuf; + H5F_istore_ud1_t udata; + herr_t ret_value = FAIL; + FUNC_ENTER(H5F_istore_write, FAIL); /* Check args */ assert(f); - assert(layout && H5D_CHUNKED == layout->type); - assert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(&(layout->addr))); + assert (offset_f); assert(size); assert(buf); - if (H5F_istore_copy_hyperslab(f, layout, H5F_ISTORE_WRITE, - offset, size, H5V_ZERO, size, - (void*)buf) < 0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "hyperslab output failure"); + /* + * For now the source must not be a hyperslab. It must be an entire + * memory buffer. + */ + for (i=0; i<layout->ndims; i++) { + offset_m[i] = 0; + size_m[i] = size[i]; + } + +#ifndef NDEBUG + for (i=0; i<layout->ndims; i++) { + assert (offset_f[i]>=0); /*negative offsets not supported*/ + assert (offset_m[i]>=0); /*negative offsets not supported*/ + assert(size[i]<MAX_SIZET); + assert(offset_m[i]+(hssize_t)size[i]<=(hssize_t)size_m[i]); + assert(layout->dim[i]>0); } - FUNC_LEAVE(SUCCEED); +#endif + + /* + * This is the general case. We set up multi-dimensional counters + * (idx_min, idx_max, and idx_cur) and loop through the chunks copying + * each chunk into a temporary buffer, compressing or decompressing, and + * then copying it to it's destination. + */ + for (i=0; i<layout->ndims; i++) { + idx_min[i] = offset_f[i] / layout->dim[i]; + idx_max[i] = (offset_f[i]+size[i]-1) / layout->dim[i] + 1; + idx_cur[i] = idx_min[i]; + } + + /* Allocate buffers */ + for (i=0, chunk_size=1; i<layout->ndims; i++) { + chunk_size *= layout->dim[i]; + } + chunk = H5MM_xmalloc(chunk_size); + if (comp && H5Z_NONE!=comp->method) { + compressed = H5MM_xmalloc (chunk_size); + } + + /* Initialize non-changing part of udata */ + udata.mesg = *layout; + + /* Loop over all chunks */ + while (1) { + + for (i=0; i<layout->ndims; i++) { + /* The location and size of the chunk being accessed */ + assert (layout->dim[i] < MAX_HSSIZET); + udata.key.offset[i] = idx_cur[i] * (hssize_t)(layout->dim[i]); + + /* The offset and size wrt the chunk */ + offset_wrt_chunk[i] = MAX(offset_f[i], udata.key.offset[i]) - + udata.key.offset[i]; + sub_size[i] = MIN((idx_cur[i]+1)*layout->dim[i], + offset_f[i]+size[i]) - + (udata.key.offset[i] + offset_wrt_chunk[i]); + + /* Offset into mem buffer */ + sub_offset_m[i] = udata.key.offset[i] + offset_wrt_chunk[i] + + offset_m[i] - offset_f[i]; + } + + /* + * If we are writing a partial chunk then load the chunk from disk + * and uncompress it if it exists. + */ + if (!H5V_vector_zerop_s(layout->ndims, offset_wrt_chunk) || + !H5V_vector_eq_u(layout->ndims, sub_size, layout->dim)) { + if (H5B_find (f, H5B_ISTORE, &(layout->addr), &udata)>=0 && + H5F_addr_defined (&(udata.addr))) { + + if (compressed && udata.key.nbytes<chunk_size) { + if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, + compressed)<0) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + if (chunk_size!=H5Z_uncompress (comp, udata.key.nbytes, + compressed, chunk_size, + chunk)) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, + "unable to uncompress data"); + } + } else { + assert (chunk_size==udata.key.nbytes); + if (H5F_block_read(f, &(udata.addr), udata.key.nbytes, + chunk)<0) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, + "unable to read raw storage chunk"); + } + } + } else { + HDmemset(chunk, 0, chunk_size); + } + } + + /* Transfer data to the chunk */ + H5V_hyper_copy(layout->ndims, sub_size, + layout->dim, offset_wrt_chunk, chunk, + size_m, sub_offset_m, buf); + + /* Compress the chunk */ + if (compressed && + (nbytes=H5Z_compress (comp, chunk_size, chunk, compressed)) && + nbytes<chunk_size) { + outbuf = compressed; + } else { + outbuf = chunk; + nbytes = chunk_size; + } + + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk + * if its size changed. Then write the data into the file. + */ + H5F_addr_undef(&(udata.addr)); + udata.key.nbytes = nbytes; + if (H5B_insert(f, H5B_ISTORE, &(layout->addr), &udata)<0) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "unable to allocate chunk"); + } + if (H5F_block_write(f, &(udata.addr), nbytes, outbuf) < 0) { + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "unable to write raw storage chunk"); + } + + /* Increment indices */ + for (i=layout->ndims-1, carry=1; i>=0 && carry; --i) { + if (++idx_cur[i]>=idx_max[i]) idx_cur[i] = idx_min[i]; + else carry = 0; + } + if (carry) break; + } + ret_value = SUCCEED; + + done: + H5MM_xfree(chunk); + H5MM_xfree (compressed); + FUNC_LEAVE(ret_value); } @@ -921,3 +1043,36 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) FUNC_LEAVE(SUCCEED); } + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_debug + * + * Purpose: Debugs a B-tree node for indexed raw data storage. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, + intn fwidth, int ndims) +{ + H5F_istore_ud1_t udata; + + FUNC_ENTER (H5F_istore_debug, FAIL); + + HDmemset (&udata, 0, sizeof udata); + udata.mesg.ndims = ndims; + + H5B_debug (f, addr, stream, indent, fwidth, H5B_ISTORE, &udata); + + FUNC_LEAVE (SUCCEED); +} diff --git a/src/H5Flow.c b/src/H5Flow.c index 5074815..faa2202 100644 --- a/src/H5Flow.c +++ b/src/H5Flow.c @@ -586,7 +586,7 @@ H5F_addr_cmp(const haddr_t *a1, const haddr_t *a2) *------------------------------------------------------------------------- */ void -H5F_addr_undef(haddr_t *addr /*out */ ) +H5F_addr_undef(haddr_t *addr/*out*/) { assert(addr); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 3439686..d9ce06d 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -462,6 +462,7 @@ typedef struct H5F_t { struct H5O_layout_t; /*forward decl for prototype arguments */ struct H5O_efl_t; /*forward decl for prototype arguments */ +struct H5O_compress_t; /*forward decl for prototype arguments */ /* library variables */ extern const H5F_create_t H5F_create_dflt; @@ -476,14 +477,18 @@ H5F_t *H5F_open(const char *name, uintn flags, herr_t H5F_close(H5F_t *f); herr_t H5F_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, intn fwidth); +herr_t H5F_istore_debug(H5F_t *f, const haddr_t *addr, FILE * stream, + intn indent, intn fwidth, int ndims); /* Functions that operate on array storage */ herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, const hsize_t _hslab_size[], const hsize_t mem_size[], const hssize_t mem_offset[], const hssize_t file_offset[], void *_buf/*out*/); herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, const hsize_t _hslab_size[], const hsize_t mem_size[], const hssize_t mem_offset[], const hssize_t file_offset[], const void *_buf); @@ -491,9 +496,11 @@ herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, /* Functions that operate on indexed storage */ herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); herr_t H5F_istore_read(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const hssize_t offset[], const hsize_t size[], void *buf /*out */ ); herr_t H5F_istore_write(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const hssize_t offset[], const hsize_t size[], const void *buf); diff --git a/src/H5Gnode.c b/src/H5Gnode.c index 5580993..9b6b749 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -82,6 +82,7 @@ H5B_class_t H5B_SNODE[1] = {{ H5G_node_iterate, /*list */ H5G_node_decode_key, /*decode */ H5G_node_encode_key, /*encode */ + NULL, /*debug key */ }}; /* Interface initialization */ @@ -128,3 +128,89 @@ H5MF_free(H5F_t __unused__ *f, const haddr_t *addr, hsize_t size) FUNC_LEAVE(SUCCEED); } + + +/*------------------------------------------------------------------------- + * Function: H5MF_realloc + * + * Purpose: Changes the size of an allocated chunk, possibly moving it to + * a new address. The chunk to change is at address ORIG_ADDR + * and is exactly ORIG_SIZE bytes (if these are zero and undef + * then this function acts like H5MF_alloc). The new size will + * be NEW_SIZE and its address is returned though NEW_ADDR (if + * NEW_SIZE is zero then this function acts like H5MF_free and + * an undefined address is returned for NEW_ADDR). + * + * If the new size is less than the old size then the new + * address will be the same as the old address (except for the + * special case where the new size is zero). + * + * If the new size is more than the old size then most likely a + * new address will be returned. However, under certain + * circumstances the library may return the same address. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_realloc (H5F_t *f, intn op, hsize_t orig_size, const haddr_t *orig_addr, + hsize_t new_size, haddr_t *new_addr/*out*/) +{ + FUNC_ENTER (H5MF_realloc, FAIL); + + if (0==orig_size) { + /* Degenerate to H5MF_alloc() */ + assert (!H5F_addr_defined (orig_addr)); + if (new_size>0) { + if (H5MF_alloc (f, op, new_size, new_addr/*out*/)<0) { + HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL, + "unable to allocate new file memory"); + } + } else { + H5F_addr_undef (new_addr); + } + + } else if (0==new_size) { + /* Degenerate to H5MF_free() */ + assert (H5F_addr_defined (orig_addr)); + if (H5MF_free (f, orig_addr, orig_size)<0) { + HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL, + "unable to free old file memory"); + } + H5F_addr_undef (new_addr); + + } else if (new_size > orig_size) { + /* Size is getting larger */ + if (H5MF_alloc (f, op, new_size, new_addr/*out*/)<0) { + HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL, + "unable to allocate new file memory"); + } + if (H5MF_free (f, orig_addr, orig_size)<0) { + HRETURN_ERROR (H5E_RESOURCE, H5E_CANTINIT, FAIL, + "unable to free old file memory"); + } + + } else { + /* New size is not larger */ +#ifdef H5MF_DEBUG + if (new_size<orig_size) { + HDfprintf (stderr, "H5MF: realloc lost %Hd bytes\n", + orig_size-new_size); + } +#endif + *new_addr = *orig_addr; + } + + FUNC_LEAVE (SUCCEED); +} + + + diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index ea80ca1..202f36c 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -39,4 +39,8 @@ */ herr_t H5MF_alloc (H5F_t *f, intn, hsize_t size, haddr_t *addr/*out*/); herr_t H5MF_free (H5F_t *f, const haddr_t *addr, hsize_t size); +herr_t H5MF_realloc (H5F_t *f, intn op, hsize_t orig_size, + const haddr_t *orig_addr, hsize_t new_size, + haddr_t *new_addr/*out*/); + #endif @@ -169,19 +169,8 @@ H5MM_xstrdup(const char *s) *------------------------------------------------------------------------- */ void * -H5MM_xfree(const void *mem) +H5MM_xfree(void *mem) { - /* - * free(3) takes a non-const pointer as an argument even though - * conceptually the argument could be a constant because by time - * free() mucks with it's contents, it should already be free :-) - * Instead of passing a const arg to free, which generates a - * compiler warning, we cast it to a non-const arg first. With - * gcc, this results in a warning only if -Wcast-qual is turned on. - */ - void *non_const_mem = (void *) mem; - - if (mem) - HDfree(non_const_mem); + if (mem) HDfree(mem); return NULL; } diff --git a/src/H5MMprivate.h b/src/H5MMprivate.h index 3fef16b..2f715e5 100644 --- a/src/H5MMprivate.h +++ b/src/H5MMprivate.h @@ -29,5 +29,5 @@ void *H5MM_xmalloc (size_t size); void *H5MM_xcalloc (intn n, size_t size); void *H5MM_xrealloc (void *mem, size_t size); char *H5MM_xstrdup (const char *s); -void *H5MM_xfree (const void *mem); +void *H5MM_xfree (void *mem); #endif @@ -61,7 +61,7 @@ static const H5O_class_t *const message_type_g[] = { H5O_LAYOUT, /*0x0008 Data Layout */ NULL, /*0x0009 Not assigned */ NULL, /*0x000A Not assigned */ - NULL, /*0x000B Data storage -- compressed object */ + H5O_COMPRESS, /*0x000B Data storage -- compressed object */ NULL, /*0x000C Attribute list */ H5O_NAME, /*0x000D Object name */ NULL, /*0x000E Object modification date and time */ diff --git a/src/H5Ocomp.c b/src/H5Ocomp.c new file mode 100644 index 0000000..6b6c2e7 --- /dev/null +++ b/src/H5Ocomp.c @@ -0,0 +1,291 @@ +/* + * Copyright (C) 1998 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke <matzke@llnl.gov> + * Wednesday, April 15, 1998 + * + * Purpose: Data compression message. + */ +#include <H5private.h> +#include <H5Eprivate.h> +#include <H5MMprivate.h> +#include <H5Oprivate.h> + +#define PABLO_MASK H5O_comp_mask + +/* PRIVATE PROTOTYPES */ +static herr_t H5O_comp_encode (H5F_t *f, uint8 *p, const void *mesg); +static void *H5O_comp_decode (H5F_t *f, const uint8 *p, H5HG_t *hobj); +static void *H5O_comp_copy (const void *_mesg, void *_dest); +static size_t H5O_comp_size (H5F_t *f, const void *_mesg); +static herr_t H5O_comp_reset (void *_mesg); +static herr_t H5O_comp_debug (H5F_t *f, const void *_mesg, + FILE * stream, intn indent, intn fwidth); + +/* This message derives from H5O */ +const H5O_class_t H5O_COMPRESS[1] = {{ + H5O_COMPRESS_ID, /* message id number */ + "compression", /* message name for debugging */ + sizeof(H5O_compress_t), /* native message size */ + H5O_comp_decode, /* decode message */ + H5O_comp_encode, /* encode message */ + H5O_comp_copy, /* copy the native value */ + H5O_comp_size, /* size of raw message */ + H5O_comp_reset, /* reset method */ + NULL, /* share method */ + H5O_comp_debug, /* debug the message */ +}}; + +/* Interface initialization */ +static hbool_t interface_initialize_g = FALSE; +#define INTERFACE_INIT NULL + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_decode + * + * Purpose: Decodes a compression message. + * + * Return: Success: Ptr to the native message. + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5O_comp_decode(H5F_t __unused__ *f, const uint8 *p, H5HG_t __unused__ *hobj) +{ + H5O_compress_t *comp = NULL; + + FUNC_ENTER(H5O_comp_decode, NULL); + + /* check args */ + assert(p); + + /* Decode */ + comp = H5MM_xcalloc(1, sizeof *comp); + comp->method = *p++; + comp->flags = *p++; + UINT16DECODE (p, comp->cd_size); + + if (comp->cd_size>0) { + comp->client_data = H5MM_xmalloc (comp->cd_size); + HDmemcpy (comp->client_data, p, comp->cd_size); + } + + FUNC_LEAVE(comp); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_encode + * + * Purpose: Encodes message MESG into buffer P. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_comp_encode (H5F_t __unused__ *f, uint8 *p/*out*/, const void *mesg) +{ + const H5O_compress_t *comp = (const H5O_compress_t*)mesg; + + FUNC_ENTER (H5O_comp_encode, FAIL); + + /* Check args */ + assert (p); + assert (mesg); + + *p++ = comp->method; + *p++ = comp->flags; + UINT16ENCODE (p, comp->cd_size); + if (comp->cd_size) { + HDmemcpy (p, comp->client_data, comp->cd_size); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_copy + * + * Purpose: Copies a compression message from SRC to DST allocating DST + * if necessary. If DST is already allocated then we assume + * that it isn't initialized. + * + * Return: Success: Ptr to DST or allocated result. + * + * Failure: NULL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5O_comp_copy (const void *_src, void *_dst/*out*/) +{ + const H5O_compress_t *src = (const H5O_compress_t *)_src; + H5O_compress_t *dst = (H5O_compress_t *)_dst; + + FUNC_ENTER (H5O_comp_copy, NULL); + + if (!dst) dst = H5MM_xmalloc (sizeof *dst); + *dst = *src; + if (src->cd_size>0) { + dst->client_data = H5MM_xmalloc (src->cd_size); + HDmemcpy (dst->client_data, src->client_data, src->cd_size); + } + + FUNC_LEAVE (dst); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_size + * + * Purpose: Determines the size of a raw compression message. + * + * Return: Success: Size of message. + * + * Failure: zero + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +H5O_comp_size (H5F_t __unused__ *f, const void *mesg) +{ + const H5O_compress_t *comp = (const H5O_compress_t*)mesg; + + FUNC_ENTER (H5O_comp_size, 0); + FUNC_LEAVE (4+comp->cd_size); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_reset + * + * Purpose: Resets a compression message by freeing the client data and + * setting all fields to zero. The MESG buffer is not freed. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_comp_reset (void *mesg) +{ + H5O_compress_t *comp = (H5O_compress_t*)mesg; + + FUNC_ENTER (H5O_comp_reset, FAIL); + + assert (comp); + H5MM_xfree (comp->client_data); + HDmemset (comp, 0, sizeof *comp); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_comp_debug + * + * Purpose: Prints debugging information for compression message MESG on + * output stream STREAM. Each line is indented INDENT + * characters and the field name takes up FWIDTH characters. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_comp_debug (H5F_t __unused__ *f, const void *mesg, FILE *stream, + intn indent, intn fwidth) +{ + const H5O_compress_t *comp = (const H5O_compress_t *)mesg; + size_t i, j; + + FUNC_ENTER(H5O_comp_debug, FAIL); + + /* check args */ + assert(f); + assert(comp); + assert(stream); + assert(indent >= 0); + assert(fwidth >= 0); + + fprintf (stream, "%*s%-*s %d\n", indent, "", fwidth, + "Method:", + (int)(comp->method)); + fprintf (stream, "%*s%-*s 0x%02x\n", indent, "", fwidth, + "Flags:", + (unsigned)(comp->flags)); + fprintf (stream, "%*s%-*s %lu\n", indent, "", fwidth, + "Size of client data:", + (unsigned long)(comp->cd_size)); + + if (comp->cd_size>0) { + fprintf (stream, "%*s%s\n", indent, "", "Client Data:"); + for (i=0; i<comp->cd_size; i+=16) { + fprintf (stream, "%*s%04d: ", indent+3, "", i); + for (j=0; j<16; j++) { + if (8==j) putc (' ', stream); + if (i+j<comp->cd_size) { + fprintf (stream, "%02x ", comp->client_data[i+j]); + } else { + fputs (" ", stream); + } + } + for (j=0; j<16 && i+j<comp->cd_size; j++) { + if (8==j) putc (' ', stream); + if (comp->client_data[i+j]>' ' && + comp->client_data[i+j]<='~') { + putc (comp->client_data[i+j], stream); + } else { + putc ('.', stream); + } + putc ('\n', stream); + } + } + } else { + fprintf (stream, "%*s%-*s None\n", indent, "", fwidth, "Client Data:"); + } + + FUNC_LEAVE(SUCCEED); +} diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 7f4fd63..d3836f4 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -26,6 +26,7 @@ #include <H5HGprivate.h> #include <H5Tprivate.h> #include <H5Sprivate.h> +#include <H5Zprivate.h> /* * Align messages on 8-byte boundaries because we would like to copy the @@ -166,6 +167,19 @@ typedef struct H5O_layout_t { } H5O_layout_t; /* + * Compression message. + */ +#define H5O_COMPRESS_ID 0x000b +extern const H5O_class_t H5O_COMPRESS[1]; + +typedef struct H5O_compress_t { + H5Z_method_t method; /*algorithm ID */ + uintn flags; /*flags */ + size_t cd_size; /*client data size */ + uint8 *client_data; /*client data passed to algorithm */ +} H5O_compress_t; + +/* * Object name message. */ #define H5O_NAME_ID 0x000d @@ -942,7 +942,7 @@ H5Pget_chunk(hid_t tid, int max_ndims, hsize_t dim[]/*out*/) if (H5P_DATASET_CREATE != H5Pget_class(tid) || NULL == (tmpl = H5I_object(tid))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, - "not a dataset creation template"); + "not a dataset creation property list"); } if (H5D_CHUNKED != tmpl->layout) { HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1847,7 +1847,178 @@ H5Pget_preserve (hid_t plist_id) FUNC_LEAVE (plist->need_bkg?TRUE:FALSE); } + + +/*------------------------------------------------------------------------- + * Function: H5Pset_compression + * + * Purpose: Sets the compression method in a dataset creation property + * list. This sets default values for the compression + * attributes by setting the flags to zero and supplying no + * compression client data. It's probably better to use + * specific compression initialization functions like + * H5Pset_deflate(). + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_compression (hid_t plist_id, H5Z_method_t method) +{ + H5D_create_t *plist = NULL; + + FUNC_ENTER (H5Pset_compression, FAIL); + + /* Check arguments */ + if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) || + NULL==(plist=H5I_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset creation property list"); + } + if (method<0 || method>H5Z_MAXVAL) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid compression method"); + } + + /* Clear any previous compression method info, then set new value */ + H5O_reset (H5O_COMPRESS, &(plist->compress)); + plist->compress.method = method; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_compression + * + * Purpose: Gets the compression method information from a dataset + * creation property list. + * + * Return: Success: Compression method. + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5Z_method_t +H5Pget_compression (hid_t plist_id) +{ + H5D_create_t *plist = NULL; + + FUNC_ENTER (H5Pget_compression, FAIL); + + /* Check arguments */ + if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) || + NULL==(plist=H5I_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset creation property list"); + } + + FUNC_LEAVE (plist->compress.method); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pset_deflate + * + * Purpose: Sets the compression method for a dataset creation property + * list to H5D_COMPRESS_DEFLATE and the compression level to + * LEVEL which should be a value between zero and nine, + * inclusive. Lower compression levels are faster but result in + * less compression. This is the same algorithm as used by the + * GNU gzip program. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_deflate (hid_t plist_id, int level) +{ + H5D_create_t *plist = NULL; + + FUNC_ENTER (H5Pset_deflate, FAIL); + + /* Check arguments */ + if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) || + NULL==(plist=H5I_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset creation property list"); + } + if (level<0 || level>9) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid deflate level"); + } + + /* Clear previous compression parameters */ + H5O_reset (H5O_COMPRESS, &(plist->compress)); + plist->compress.method = H5Z_DEFLATE; + plist->compress.flags = level; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_deflate + * + * Purpose: Returns the deflate compression level from a dataset creation + * property list that uses that method. + * + * Return: Success: A value between zero and nine, inclusive. + * Smaller values indicate faster compression + * while higher values indicate better + * compression ratios. + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +int +H5Pget_deflate (hid_t plist_id) +{ + H5D_create_t *plist = NULL; + + FUNC_ENTER (H5Pget_deflate, FAIL); + /* Check arguments */ + if (H5P_DATASET_CREATE!=H5Pget_class (plist_id) || + NULL==(plist=H5I_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset creation property list"); + } + if (H5Z_DEFLATE!=plist->compress.method) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "deflate method is not being used"); + } + + FUNC_LEAVE (plist->compress.flags % 10); +} #ifdef HAVE_PARALLEL diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index ad7f98b..bd87342 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -25,6 +25,7 @@ #include <H5Ipublic.h> #include <H5Dpublic.h> #include <H5Fpublic.h> +#include <H5Zpublic.h> /* Template classes */ typedef enum H5P_class_t { @@ -87,6 +88,10 @@ herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg); size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/); herr_t H5Pset_preserve (hid_t plist_id, hbool_t status); int H5Pget_preserve (hid_t plist_id); +herr_t H5Pset_compression (hid_t plist_id, H5Z_method_t method); +H5Z_method_t H5Pget_compression (hid_t plist_id); +herr_t H5Pset_deflate (hid_t plist_id, int level); +int H5Pget_deflate (hid_t plist_id); #ifdef HAVE_PARALLEL herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info); diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index db6bf9e..07f4108 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -70,6 +70,7 @@ typedef struct H5S_tconv_t { /* Gather elements from disk to type conversion buffer */ size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *tconv_buf/*out*/); @@ -86,18 +87,21 @@ typedef struct H5S_tconv_t { /* Scatter elements from type conversion buffer to disk */ herr_t (*fscat)(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *compress, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, const void *tconv_buf); /* Read from file to application w/o intermediate scratch buffer */ herr_t (*read)(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, void *buf/*out*/); /* Write directly from app buffer to file */ herr_t (*write)(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, const void *buf); @@ -125,6 +129,7 @@ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, const H5S_t *file_space, size_t desired_nelmts, H5S_number_t *numbering/*out*/); size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *tconv_buf/*out*/); @@ -135,14 +140,17 @@ size_t H5S_simp_mgath (const void *buf, size_t elmt_size, const H5S_t *mem_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *tconv_buf/*out*/); herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, const void *tconv_buf); herr_t H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, void *buf/*out*/); herr_t H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, const void *buf); diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c index 976840f..f8b3e0d 100644 --- a/src/H5Ssimp.c +++ b/src/H5Ssimp.c @@ -124,7 +124,7 @@ H5S_simp_init (const struct H5O_layout_t __unused__ *layout, */ size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t __unused__ *numbering, size_t start, size_t nelmts, void *buf/*out*/) @@ -184,7 +184,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, /* * Gather from file. */ - if (H5F_arr_read (f, layout, efl, hsize, hsize, zero, file_offset, + if (H5F_arr_read (f, layout, comp, efl, hsize, hsize, zero, file_offset, buf/*out*/)<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_READERROR, 0, "read error"); } @@ -408,7 +408,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, */ herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t __unused__ *numbering, size_t start, size_t nelmts, const void *buf) @@ -468,7 +468,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, /* * Scatter to file. */ - if (H5F_arr_write (f, layout, efl, hsize, hsize, zero, + if (H5F_arr_write (f, layout, comp, efl, hsize, hsize, zero, file_offset, buf)<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); } @@ -499,9 +499,9 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, */ herr_t H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - void *buf/*out*/) + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, + size_t elmt_size, const H5S_t *file_space, + const H5S_t *mem_space, void *buf/*out*/) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; hsize_t hslab_size[H5O_LAYOUT_NDIMS]; @@ -573,7 +573,7 @@ H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout, mem_offset[file_space->u.simple.rank] = 0; /* Read the hyperslab */ - if (H5F_arr_read (f, layout, efl, hslab_size, + if (H5F_arr_read (f, layout, comp, efl, hslab_size, mem_size, mem_offset, file_offset, buf)<0) { HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, "unable to read dataset"); } @@ -604,9 +604,9 @@ H5S_simp_read (H5F_t *f, const struct H5O_layout_t *layout, */ herr_t H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout, - const struct H5O_efl_t *efl, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - const void *buf) + const struct H5O_compress_t *comp, const struct H5O_efl_t *efl, + size_t elmt_size, const H5S_t *file_space, + const H5S_t *mem_space, const void *buf) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; hsize_t hslab_size[H5O_LAYOUT_NDIMS]; @@ -678,7 +678,7 @@ H5S_simp_write (H5F_t *f, const struct H5O_layout_t *layout, mem_offset[file_space->u.simple.rank] = 0; /* Write the hyperslab */ - if (H5F_arr_write (f, layout, efl, hslab_size, + if (H5F_arr_write (f, layout, comp, efl, hslab_size, mem_size, mem_offset, file_offset, buf)<0) { HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to write dataset"); @@ -231,15 +231,17 @@ H5T_init_interface(void) * Register conversion functions beginning with the most general and * ending with the most specific. */ - if (H5Tregister_soft(H5T_INTEGER, H5T_INTEGER, H5T_conv_order) < 0) { + if (H5Tregister_soft("ibo", H5T_INTEGER, H5T_INTEGER, + H5T_conv_order) < 0) { HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to register conversion function"); } - if (H5Tregister_soft(H5T_FLOAT, H5T_FLOAT, H5T_conv_order) < 0) { + if (H5Tregister_soft("fbo", H5T_FLOAT, H5T_FLOAT, H5T_conv_order) < 0) { HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to register conversion function"); } - if (H5Tregister_soft (H5T_COMPOUND, H5T_COMPOUND, H5T_conv_struct)<0) { + if (H5Tregister_soft ("struct", H5T_COMPOUND, H5T_COMPOUND, + H5T_conv_struct)<0) { HRETURN_ERROR (H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to register conversion function"); } @@ -268,10 +270,14 @@ H5T_init_interface(void) static void H5T_term_interface(void) { - int i; + intn i; H5T_path_t *path = NULL; H5T_cdata_t *pcdata = NULL; H5T_conv_t cfunc = NULL; +#ifdef H5T_DEBUG + intn nprint=0; + hsize_t nbytes; +#endif /* Unregister all conversion functions */ for (i=0; i<H5T_npath_g; i++) { @@ -286,6 +292,29 @@ H5T_term_interface(void) #endif H5E_clear(); /*ignore the error*/ } +#ifdef H5T_DEBUG + if (0==nprint++) { + HDfprintf (stderr, "H5T: type conversion statistics " + "accumulated over life of library:\n"); + HDfprintf (stderr, " %-*s %8s/%-5s %8s %8s %8s %15s\n", + H5T_NAMELEN-1, "Name", "Elmts", "Calls", "User", + "System", "Elapsed", "Bandwidth"); + HDfprintf (stderr, " %-*s %8s-%-5s %8s %8s %8s %15s\n", + H5T_NAMELEN-1, "----", "-----", "-----", "----", + "------", "-------", "---------"); + } + nbytes = MAX (H5T_get_size (path->src), H5T_get_size (path->dst)); + nbytes *= path->cdata.stats->nelmts; + HDfprintf (stderr, " %-*s %8Hd/%-5d %8.2f %8.2f %8.2f %15g\n", + H5T_NAMELEN-1, path->name, + path->cdata.stats->nelmts, + path->cdata.stats->ncalls, + path->cdata.stats->timer.utime, + path->cdata.stats->timer.stime, + path->cdata.stats->timer.etime, + nbytes / path->cdata.stats->timer.etime); +#endif + H5MM_xfree (path->cdata.stats); } } @@ -2276,7 +2305,7 @@ H5Tis_shared (hid_t loc_id, hid_t type_id) *------------------------------------------------------------------------- */ herr_t -H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func) +H5Tregister_hard(const char *name, hid_t src_id, hid_t dst_id, H5T_conv_t func) { H5T_t *src = NULL; H5T_t *dst = NULL; @@ -2286,6 +2315,10 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func) FUNC_ENTER(H5Tregister_hard, FAIL); /* Check args */ + if (!name || !*name) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "conversion must have a name for debugging"); + } if (H5_DATATYPE != H5I_group(src_id) || NULL == (src = H5I_object(src_id)) || H5_DATATYPE != H5I_group(dst_id) || @@ -2294,7 +2327,7 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func) } /* Locate or create a new conversion path */ - if (NULL == (path = H5T_path_find(src, dst, TRUE, func))) { + if (NULL == (path = H5T_path_find(name, src, dst, TRUE, func))) { HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to locate/allocate conversion path"); } @@ -2334,7 +2367,8 @@ H5Tregister_hard(hid_t src_id, hid_t dst_id, H5T_conv_t func) *------------------------------------------------------------------------- */ herr_t -H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func) +H5Tregister_soft(const char *name, H5T_class_t src_cls, H5T_class_t dst_cls, + H5T_conv_t func) { intn i; hid_t src_id, dst_id; @@ -2343,6 +2377,10 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func) FUNC_ENTER(H5Tregister_soft, FAIL); /* Check args */ + if (!name || !*name) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "conversion must have a name for debugging"); + } if (src_cls < 0 || src_cls >= H5T_NCLASSES || dst_cls < 0 || dst_cls >= H5T_NCLASSES) { HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -2359,6 +2397,8 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func) H5T_soft_g = H5MM_xrealloc(H5T_soft_g, H5T_asoft_g * sizeof(H5T_soft_t)); } + HDstrncpy (H5T_soft_g[H5T_nsoft_g].name, name, H5T_NAMELEN); + H5T_soft_g[H5T_nsoft_g].name[H5T_NAMELEN-1] = '\0'; H5T_soft_g[H5T_nsoft_g].src = src_cls; H5T_soft_g[H5T_nsoft_g].dst = dst_cls; H5T_soft_g[H5T_nsoft_g].func = func; @@ -2387,6 +2427,7 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func) HDmemset (&cdata, 0, sizeof cdata); cdata.command = H5T_CONV_INIT; + cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t)); if ((func) (src_id, dst_id, &cdata, 0, NULL, NULL) >= 0) { /* * Free resources used by the previous conversion function. We @@ -2403,9 +2444,14 @@ H5Tregister_soft(H5T_class_t src_cls, H5T_class_t dst_cls, H5T_conv_t func) #endif H5E_clear(); } + H5MM_xfree (path->cdata.stats); } + HDstrncpy (path->name, name, H5T_NAMELEN); + path->name[H5T_NAMELEN-1] = '\0'; path->func = func; path->cdata = cdata; + } else { + H5MM_xfree (cdata.stats); } /* Release temporary atoms */ @@ -2475,6 +2521,7 @@ H5Tunregister(H5T_conv_t func) #endif H5E_clear(); } + H5MM_xfree (path->cdata.stats); HDmemset (&(path->cdata), 0, sizeof(H5T_cdata_t)); /* @@ -2500,8 +2547,10 @@ H5Tunregister(H5T_conv_t func) } path->cdata.command = H5T_CONV_INIT; + path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t)); if ((H5T_soft_g[j].func)(src_id, dst_id, &(path->cdata), 0, NULL, NULL) >= 0) { + HDstrcpy (path->name, H5T_soft_g[j].name); path->func = H5T_soft_g[j].func; } else { H5E_clear(); @@ -3348,6 +3397,9 @@ H5T_cmp(const H5T_t *dt1, const H5T_t *dt2) * H5T_BKG_YES then a pointer to the H5T_conv_noop() function is * returned. * + * NAME is assigned to the conversion path if the path is + * created. The name is only for debugging. + * * Return: Success: A pointer to an appropriate conversion * function. The PCDATA argument is initialized * to point to type conversion data which should @@ -3378,9 +3430,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg, HRETURN(H5T_conv_noop); } - /* Find it */ - if (NULL == (path = H5T_path_find(src, dst, TRUE, NULL))) { + if (NULL == (path = H5T_path_find(NULL, src, dst, TRUE, NULL))) { HRETURN_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to create conversion path"); } @@ -3403,6 +3454,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg, * is created. If FUNC is non-null then it is registered as the * hard function for that path. * + * If a path is created then NAME is used for debugging. + * * Return: Success: Pointer to the path, valid until the path * database is modified. * @@ -3416,8 +3469,8 @@ H5T_find(const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg, *------------------------------------------------------------------------- */ H5T_path_t * -H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create, - H5T_conv_t func) +H5T_path_find(const char *name, const H5T_t *src, const H5T_t *dst, + hbool_t create, H5T_conv_t func) { intn lt = 0; /*left edge (inclusive) */ intn rt = H5T_npath_g; /*right edge (exclusive) */ @@ -3471,9 +3524,12 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create, /* Associate a function with the path if possible */ if (func) { + HDstrncpy (path->name, name, H5T_NAMELEN); + path->name[H5T_NAMELEN-1] = '\0'; path->func = func; path->is_hard = TRUE; path->cdata.command = H5T_CONV_INIT; + path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t)); if ((src_id=H5I_register(H5_DATATYPE, H5T_copy(path->src))) < 0 || (dst_id=H5I_register(H5_DATATYPE, H5T_copy(path->dst))) < 0) { HRETURN_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, @@ -3503,11 +3559,13 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create, "unable to register conv types for query"); } path->cdata.command = H5T_CONV_INIT; + path->cdata.stats = H5MM_xcalloc (1, sizeof(H5T_stats_t)); if ((H5T_soft_g[i].func) (src_id, dst_id, &(path->cdata), H5T_CONV_INIT, NULL, NULL) < 0) { HDmemset (&(path->cdata), 0, sizeof(H5T_cdata_t)); H5E_clear(); /*ignore the error*/ } else { + HDstrcpy (path->name, H5T_soft_g[i].name); path->func = H5T_soft_g[i].func; } H5I_dec_ref(src_id); @@ -3518,6 +3576,59 @@ H5T_path_find(const H5T_t *src, const H5T_t *dst, hbool_t create, FUNC_LEAVE(path); } + +/*------------------------------------------------------------------------- + * Function: H5T_timer_begin + * + * Purpose: Start a timer for a data type conversion. + * + * Return: void + * + * Programmer: Robb Matzke + * Friday, April 17, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +H5T_timer_begin (H5_timer_t *timer, H5T_cdata_t *cdata) +{ + assert (timer); + assert (cdata); + assert (cdata->stats); + + H5_timer_begin (timer); +} + + +/*------------------------------------------------------------------------- + * Function: H5T_timer_end + * + * Purpose: Ends a timer for a data type conversion + * + * Return: void + * + * Programmer: Robb Matzke + * Friday, April 17, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +H5T_timer_end (H5_timer_t *timer, H5T_cdata_t *cdata, size_t nelmts) +{ + assert (timer); + assert (cdata); + assert (cdata->stats); + + H5_timer_end (&(cdata->stats->timer), timer); + cdata->stats->ncalls++; + cdata->stats->nelmts += nelmts; +} + + /*------------------------------------------------------------------------- * Function: H5T_debug * diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 0e53d18..be07341 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -62,14 +62,8 @@ H5T_conv_noop(hid_t __unused__ src_id, hid_t __unused__ dst_id, case H5T_CONV_FREE: /* Nothing to free */ -#ifdef H5T_DEBUG - if (cdata->ncalls>0) { - fprintf (stderr, "H5T: H5T_conv_noop statistics...\n"); - fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls); - fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts); - } -#endif - + break; + default: HRETURN_ERROR (H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown conversion command"); @@ -173,13 +167,6 @@ H5T_conv_order(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, case H5T_CONV_FREE: /* Free private data */ -#ifdef H5T_DEBUG - if (cdata->ncalls>0) { - fprintf (stderr, "H5T: H5T_conv_order statistics...\n"); - fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls); - fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts); - } -#endif break; default: @@ -396,13 +383,6 @@ H5T_conv_struct(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, /* * Free the private conversion data. */ -#ifdef H5T_DEBUG - if (cdata->ncalls>0) { - fprintf (stderr, "H5T: H5T_conv_struct statistics...\n"); - fprintf (stderr, " Number of calls: %lu\n", cdata->ncalls); - fprintf (stderr, " Data points converted: %lu\n", cdata->nelmts); - } -#endif H5MM_xfree (priv->src2dst); H5MM_xfree (priv->src_memb_id); H5MM_xfree (priv->dst_memb_id); diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index e6eed20..64d0fba 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -26,6 +26,8 @@ #include <H5HGprivate.h> #include <H5Tprivate.h> +#define H5T_NAMELEN 10 /*length of debugging name buffer */ + typedef struct H5T_atomic_t { H5T_order_t order; /*byte order */ size_t prec; /*precision in bits */ @@ -84,6 +86,7 @@ typedef struct H5T_member_t { /* The data type conversion database */ typedef struct H5T_path_t { + char name[H5T_NAMELEN]; /*name for debugging only */ H5T_t *src; /*source data type ID */ H5T_t *dst; /*destination data type ID */ H5T_conv_t func; /*data conversion function */ @@ -93,14 +96,24 @@ typedef struct H5T_path_t { /* The master list of soft conversion functions */ typedef struct H5T_soft_t { + char name[H5T_NAMELEN]; /*name for debugging only */ H5T_class_t src; /*source data type class */ H5T_class_t dst; /*destination data type class */ H5T_conv_t func; /*the conversion function */ } H5T_soft_t; +/* Statistics about a conversion function */ +typedef struct H5T_stats_t { + uintn ncalls; /*num calls to conversion function */ +#ifdef H5T_DEBUG + hsize_t nelmts; /*total data points converted */ + H5_timer_t timer; /*total time for conversion */ +#endif +} H5T_stats_t; + /* Function prototypes for H5T package scope */ -H5T_path_t *H5T_path_find (const H5T_t *src, const H5T_t *dst, - hbool_t create, H5T_conv_t func); +H5T_path_t *H5T_path_find (const char *name, const H5T_t *src, + const H5T_t *dst, hbool_t create, H5T_conv_t func); /* Conversion functions */ herr_t H5T_conv_order (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index 388c20a..a9241d6 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -45,6 +45,8 @@ herr_t H5T_pack (H5T_t *dt); herr_t H5T_debug (H5T_t *dt, FILE * stream); H5T_conv_t H5T_find (const H5T_t *src, const H5T_t *dst, H5T_bkg_t need_bkg, H5T_cdata_t **pcdata/*out*/); +void H5T_timer_begin (H5_timer_t *timer, H5T_cdata_t *cdata); +void H5T_timer_end (H5_timer_t *timer, H5T_cdata_t *cdata, size_t nelmts); /* * This conversion function is here so we can determine whether a conversion diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index c17ac3a..4c1a680 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -111,9 +111,8 @@ typedef struct H5T_cdata_t { H5T_cmd_t command;/*what should the conversion function do? */ H5T_bkg_t need_bkg;/*is the background buffer needed? */ hbool_t recalc; /*recalculate private data */ - unsigned long ncalls; /*number of calls to conversion function */ - unsigned long nelmts; /*total number of data points converted */ void *priv; /*private data */ + struct H5T_stats_t *stats; /*statistics for the conversion */ } H5T_cdata_t; /* All data type conversion functions are... */ @@ -232,8 +231,10 @@ herr_t H5Tset_cset (hid_t type_id, H5T_cset_t cset); herr_t H5Tset_strpad (hid_t type_id, H5T_str_t strpad); /* Type conversion database */ -herr_t H5Tregister_hard (hid_t src_id, hid_t dst_id, H5T_conv_t func); -herr_t H5Tregister_soft (H5T_class_t src, H5T_class_t dst, H5T_conv_t func); +herr_t H5Tregister_hard (const char *name, hid_t src_id, hid_t dst_id, + H5T_conv_t func); +herr_t H5Tregister_soft (const char *name, H5T_class_t src, H5T_class_t dst, + H5T_conv_t func); herr_t H5Tunregister (H5T_conv_t func); H5T_conv_t H5Tfind (hid_t src_id, hid_t dst_id, H5T_cdata_t **pcdata); diff --git a/src/H5Z.c b/src/H5Z.c new file mode 100644 index 0000000..993cc5e --- /dev/null +++ b/src/H5Z.c @@ -0,0 +1,501 @@ +/* + * Copyright (C) 1998 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke <matzke@llnl.gov> + * Thursday, April 16, 1998 + * + * Purpose: Functions for data compression. + */ +#include <H5private.h> +#include <H5Eprivate.h> +#include <H5MMprivate.h> +#include <H5Oprivate.h> +#include <H5Zprivate.h> + +#ifdef HAVE_ZLIB_H +# include <zlib.h> +#endif + +/* Interface initialization */ +#define PABLO_MASK H5Z_mask +#define INTERFACE_INIT H5Z_init_interface +static intn interface_initialize_g = FALSE; +static herr_t H5Z_init_interface (void); +static void H5Z_term_interface (void); + +/* + * The compression table maps compression method number to a struct that + * contains pointers to the compress and uncompress methods along with timing + * statistics. + */ +typedef struct H5Z_class_t { + char *name; /*method name for debugging */ + H5Z_func_t compress; /*compression function */ + H5Z_func_t uncompress; /*uncompression function */ + +#ifdef H5Z_DEBUG + struct { + hsize_t nbytes; /*bytes compressed including overruns */ + hsize_t over; /*bytes of overrun */ + H5_timer_t timer; /*total compression time inc. overruns */ + hsize_t failed; /*bytes of failure (not overruns) */ + } comp; + + struct { + hsize_t nbytes; /*bytes uncompressed, including overruns*/ + hsize_t over; /*bytes of overrun */ + H5_timer_t timer; /*total uncompression time */ + hsize_t failed; /*bytes of failure (not overruns) */ + } uncomp; +#endif +} H5Z_class_t; +static H5Z_class_t H5Z_g[H5Z_MAXVAL+1]; + +/* Compression and uncompression methods */ +static size_t H5Z_zlib_c (unsigned int flags, size_t __unused__ cd_size, + const void __unused__ *client_data, + size_t src_nbytes, const void *_src, + size_t dst_nbytes, void *dst/*out*/); +static size_t H5Z_zlib_u (unsigned int flags, size_t __unused__ cd_size, + const void __unused__ *client_data, + size_t src_nbytes, const void *_src, + size_t dst_nbytes, void *dst/*out*/); + + +/*------------------------------------------------------------------------- + * Function: H5Z_init_interface + * + * Purpose: Initializes the data compression layer. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_init_interface (void) +{ + FUNC_ENTER (H5Z_init_interface, FAIL); + + H5_add_exit (H5Z_term_interface); + + H5Z_register (H5Z_NONE, "none", NULL, NULL); + H5Z_register (H5Z_DEFLATE, "deflate", H5Z_zlib_c, H5Z_zlib_u); + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_term_interface + * + * Purpose: Terminate the H5Z layer. + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void +H5Z_term_interface (void) +{ +#ifdef H5Z_DEBUG + int i, nprint=0; + char name[16]; + + for (i=0; i<=H5Z_MAXVAL; i++) { + if (H5Z_g[i].comp.nbytes || H5Z_g[i].uncomp.nbytes) { + if (0==nprint++) { + HDfprintf (stderr, "H5Z: compression statistics accumulated " + "over life of library:\n"); + HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n", + "Method", "Total", "Overrun", "Errors", "User", + "System", "Elapsed", "Bandwidth"); + HDfprintf (stderr, " %-10s %8s %8s %8s %8s %8s %8s %9s\n", + "------", "-----", "-------", "------", "----", + "------", "-------", "---------"); + } + sprintf (name, "%s-c", H5Z_g[i].name); + HDfprintf (stderr, + " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ", + name, + H5Z_g[i].comp.nbytes, + H5Z_g[i].comp.over, + H5Z_g[i].comp.failed, + H5Z_g[i].comp.timer.utime, + H5Z_g[i].comp.timer.stime, + H5Z_g[i].comp.timer.etime); + if (H5Z_g[i].comp.timer.etime>0) { + HDfprintf (stderr, "%9.3e\n", + H5Z_g[i].comp.nbytes / H5Z_g[i].comp.timer.etime); + } else { + HDfprintf (stderr, "%9s\n", "NaN"); + } + + sprintf (name, "%s-u", H5Z_g[i].name); + HDfprintf (stderr, + " %-12s %8Hd %8Hd %8Hd %8.2f %8.2f %8.2f ", + name, + H5Z_g[i].uncomp.nbytes, + H5Z_g[i].uncomp.over, + H5Z_g[i].uncomp.failed, + H5Z_g[i].uncomp.timer.utime, + H5Z_g[i].uncomp.timer.stime, + H5Z_g[i].uncomp.timer.etime); + if (H5Z_g[i].uncomp.timer.etime>0) { + HDfprintf (stderr, "%9.3e\n", + H5Z_g[i].uncomp.nbytes/H5Z_g[i].uncomp.timer.etime); + } else { + HDfprintf (stderr, "%9s\n", "NaN"); + } + } + } +#endif +} + + +/*------------------------------------------------------------------------- + * Function: H5Zregister + * + * Purpose: This function registers new compression and uncompression + * methods for a method number. The NAME argument is used for + * debugging and may be the null pointer. Either or both of + * CFUNC (the compression function) and UFUNC (the uncompression + * method) may be null pointers. + * + * The statistics associated with a method number are not reset + * by this function; they accumulate over the life of the + * library. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Zregister (H5Z_method_t method, const char *name, H5Z_func_t cfunc, + H5Z_func_t ufunc) +{ + FUNC_ENTER (H5Zregister, FAIL); + + /* Check args */ + if (method<0 || method>H5Z_MAXVAL) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid data compression method number"); + } + if (method<16) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to modify predefined compression methods"); + } + + /* Do it */ + if (H5Z_register (method, name, cfunc, ufunc)<0) { + HRETURN_ERROR (H5E_COMP, H5E_CANTINIT, FAIL, + "unable to register compression methods"); + } + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_compress + * + * Purpose: Compress NBYTES from SRC into at most NBYTES of DST. + * + * Return: Success: Number of bytes in DST + * + * Failure: 0 if the DST buffer overflowed or something + * else went wrong. + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5Z_compress (const H5O_compress_t *comp, size_t nbytes, const void *src, + void *dst/*out*/) +{ + size_t ret_value = 0; + H5Z_method_t method = comp ? comp->method : H5Z_NONE; + H5Z_func_t cfunc = NULL; +#ifdef H5Z_DEBUG + intn over = 0; + H5_timer_t timer; +#endif + + FUNC_ENTER (H5Z_compress, 0); + +#ifdef H5Z_DEBUG + H5_timer_begin (&timer); +#endif + + if (H5Z_NONE==method) { + /* No compression method */ + HGOTO_DONE (0); + + } else if (NULL==(cfunc=H5Z_g[method].compress)) { + /* No compress function */ + HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0, + "compression method is not supported"); + + } else if (0==(ret_value=(cfunc)(comp->flags, comp->cd_size, + comp->client_data, nbytes, + src, nbytes, dst))) { + /* Compress failed */ + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "compression failed"); + + } else if (ret_value>=nbytes) { + /* Output is not smaller than input */ +#ifdef H5Z_DEBUG + H5Z_g[method].comp.over += 1; + over = 1; +#endif + HGOTO_DONE (0); + } + + done: +#ifdef H5Z_DEBUG + H5Z_g[method].comp.nbytes += nbytes; + if (0==ret_value && !over) H5Z_g[method].comp.failed += nbytes; + H5_timer_end (&(H5Z_g[method].comp.timer), &timer); +#endif + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_uncompress + * + * Purpose: Uncompress SRC_NBYTES from SRC into at most DST_NBYTES of + * DST. + * + * Return: Success: Number of bytes in DST buffer. + * + * Failure: 0 if the uncompression failed or DST wasn't + * big enough to hold the result. + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5Z_uncompress (const H5O_compress_t *comp, size_t src_nbytes, const void *src, + size_t dst_nbytes, void *dst/*out*/) +{ + size_t ret_value = 0; + H5Z_func_t ufunc = NULL; + H5Z_method_t method = comp ? comp->method : H5Z_NONE; +#ifdef H5Z_DEBUG + H5_timer_t timer; +#endif + + FUNC_ENTER (H5Z_uncompress, 0); + +#ifdef H5Z_DEBUG + H5_timer_begin (&timer); +#endif + + if (H5Z_NONE==method) { + /* No compression method */ + assert (src_nbytes<=dst_nbytes); + HDmemcpy (dst, src, src_nbytes); + ret_value = src_nbytes; + + } else if (src_nbytes==dst_nbytes) { + /* Data is not compressed */ +#ifdef H5Z_DEBUG + H5Z_g[method].uncomp.over += src_nbytes; +#endif + HDmemcpy (dst, src, src_nbytes); + ret_value = src_nbytes; + + } else if (NULL==(ufunc=H5Z_g[method].uncompress)) { + /* No uncompress function */ + HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0, + "uncompression method is not supported"); + + } else if (0==(ret_value=(ufunc)(comp->flags, comp->cd_size, + comp->client_data, src_nbytes, + src, dst_nbytes, dst))) { + /* Uncompress failed */ + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "uncompression failed"); + } + + done: +#ifdef H5Z_DEBUG + H5Z_g[method].uncomp.nbytes += dst_nbytes; + if (0==ret_value) H5Z_g[method].uncomp.failed += dst_nbytes; + H5_timer_end (&(H5Z_g[method].uncomp.timer), &timer); +#endif + + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_register + * + * Purpose: Same as the public version except this one allows compression + * methods to be set for predefined method numbers <16. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Z_register (H5Z_method_t method, const char *name, H5Z_func_t cfunc, + H5Z_func_t ufunc) +{ + FUNC_ENTER (H5Z_register, FAIL); + + assert (method>=0 && method<=H5Z_MAXVAL); + H5MM_xfree (H5Z_g[method].name); + H5Z_g[method].name = H5MM_xstrdup (name); + H5Z_g[method].compress = cfunc; + H5Z_g[method].uncompress = ufunc; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_zlib_c + * + * Purpose: Compress SRC_NBYTES bytes from SRC into at most DST_NBYTES of + * DST using the compression level as specified in FLAGS. + * + * Return: Success: Number of bytes compressed into DST limited + * by DST_NBYTES. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z_zlib_c (unsigned int flags, size_t __unused__ cd_size, + const void __unused__ *client_data, size_t src_nbytes, + const void *src, size_t dst_nbytes, void *dst/*out*/) +{ + size_t ret_value = 0; +#ifdef HAVE_ZLIB_H + const Bytef *z_src = (const Bytef*)src; + Bytef *z_dst = (Bytef*)dst; + uLongf z_dst_nbytes = (uLongf)dst_nbytes; + uLong z_src_nbytes = (uLong)src_nbytes; + int level = flags % 10; + int status; +#endif + + FUNC_ENTER (H5Z_zlib_c, 0); + +#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H) + status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, level); + if (Z_BUF_ERROR==status) { + ret_value = dst_nbytes; + } else if (Z_MEM_ERROR==status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate memory error"); + } else if (Z_OK!=status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate error"); + } else { + ret_value = z_dst_nbytes; + } +#else + HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0, + "hdf5 was not compiled with zlib-1.0.2 or better"); +#endif + + done: + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_zlib_u + * + * Purpose: Uncompress SRC_NBYTES from SRC into at most DST_NBYTES of + * DST. + * + * Return: Success: Number of bytes returned in DST. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z_zlib_u (unsigned int __unused__ flags, size_t __unused__ cd_size, + const void __unused__ *client_data, size_t src_nbytes, + const void *src, size_t dst_nbytes, void *dst/*out*/) +{ + size_t ret_value = 0; +#ifdef HAVE_ZLIB_H + const Bytef *z_src = (const Bytef*)src; + Bytef *z_dst = (Bytef*)dst; + uLongf z_dst_nbytes = (uLongf)dst_nbytes; + uLong z_src_nbytes = (uLong)src_nbytes; + int status; +#endif + + FUNC_ENTER (H5Z_zlib_u, 0); + +#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H) + status = uncompress (z_dst, &z_dst_nbytes, z_src, z_src_nbytes); + if (Z_BUF_ERROR==status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, + "deflate destination buffer was too small"); + } else if (Z_MEM_ERROR==status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate memory error"); + } else if (Z_DATA_ERROR==status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate corrupted data"); + } else if (Z_OK!=status) { + HGOTO_ERROR (H5E_COMP, H5E_CANTINIT, 0, "deflate error"); + } + ret_value = z_dst_nbytes; +#else + HGOTO_ERROR (H5E_COMP, H5E_UNSUPPORTED, 0, + "hdf5 was not compiled with zlib-1.0.2 or better"); +#endif + + done: + FUNC_LEAVE (ret_value); +} diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h new file mode 100644 index 0000000..288ec07 --- /dev/null +++ b/src/H5Zprivate.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 1998 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke <matzke@llnl.gov> + * Thursday, April 16, 1998 + */ +#ifndef _H5Zprivate_H +#define _H5Zprivate_H + +#include <H5Zpublic.h> + +struct H5O_compress_t; /*forward decl*/ + +herr_t H5Z_register (H5Z_method_t method, const char *name, + H5Z_func_t compress, H5Z_func_t uncompress); +size_t H5Z_compress (const struct H5O_compress_t *compress, size_t nbytes, + const void *src, void *dst/*out*/); +size_t H5Z_uncompress (const struct H5O_compress_t *compress, + size_t src_nbytes, const void *src, size_t dst_nbytes, + void *dst/*out*/); + +#endif diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h new file mode 100644 index 0000000..daebf89 --- /dev/null +++ b/src/H5Zpublic.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 1998 NCSA + * All rights reserved. + * + * Programmer: Robb Matzke <matzke@llnl.gov> + * Thursday, April 16, 1998 + */ +#ifndef _H5Zpublic_H +#define _H5Zpublic_H + +/* + * Compression methods. Method zero means no compression. Methods 1 through + * 15 are defined by the library. Methods 16-255 are user-defined. + */ +typedef int H5Z_method_t; +#define H5Z_NONE 0 /*no compression, must be zero */ +#define H5Z_DEFLATE 1 /*deflation like gzip */ +#define H5Z_RES_2 2 /*reserved for internal use */ +#define H5Z_RES_3 3 /*reserved for internal use */ +#define H5Z_RES_4 4 /*reserved for internal use */ +#define H5Z_RES_5 5 /*reserved for internal use */ +#define H5Z_RES_6 6 /*reserved for internal use */ +#define H5Z_RES_7 7 /*reserved for internal use */ +#define H5Z_RES_8 8 /*reserved for internal use */ +#define H5Z_RES_9 9 /*reserved for internal use */ +#define H5Z_RES_10 10 /*reserved for internal use */ +#define H5Z_RES_11 11 /*reserved for internal use */ +#define H5Z_RES_12 12 /*reserved for internal use */ +#define H5Z_RES_13 13 /*reserved for internal use */ +#define H5Z_RES_14 14 /*reserved for internal use */ +#define H5Z_RES_15 15 /*reserved for internal use */ +/* user-defined 16-255 */ +#define H5Z_MAXVAL 255 /*maximum compression method ID */ + +/* + * A compression function takes some configuration data which comes from the + * compression message, namely FLAGS, CD_SIZE, and CLIENT_DATA. It should + * read SRC_NBYTES from SRC and compress them into at most DST_NBYTES of DST. + * If the compressed data would be larger than DST_NBYTES the function should + * return a value greater than or equal to DST_NBYTES. On failure the + * function may return zero. + * + * The uncompression function is the inverse of compression and takes the + * same arguments. The SRC_NBYTES argument is the number of compressed bytes + * in SRC. The function should uncompress SRC into DST. For redundancy, + * DST_NBYTES contains the size of the DST buffer although if the algorithm + * is operating properly and the file has not been corrupted the uncompressed + * data will never be larger than DST_NBYTES. The function should return the + * number of bytes in the DST buffer or zero on failure. Failure includes + * the overflow of the DST buffer. + */ +typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_size, + const void *client_data, size_t src_nbytes, + const void *src, size_t dst_nbytes, + void *dst/*out*/); + + +#ifdef __cplusplus +extern "C" { +#endif + +herr_t H5Zregister (H5Z_method_t method, const char *name, H5Z_func_t compress, + H5Z_func_t uncompress); + +#ifdef __cplusplus +} +#endif +#endif + diff --git a/src/H5config.h.in b/src/H5config.h.in index a715025..9bb277a 100644 --- a/src/H5config.h.in +++ b/src/H5config.h.in @@ -64,6 +64,9 @@ /* Define if you have the getpwuid function. */ #undef HAVE_GETPWUID +/* Define if you have the getrusage function. */ +#undef HAVE_GETRUSAGE + /* Define if you have the lseek64 function. */ #undef HAVE_LSEEK64 @@ -73,6 +76,9 @@ /* Define if you have the <unistd.h> header file. */ #undef HAVE_UNISTD_H +/* Define if you have the <zlib.h> header file. */ +#undef HAVE_ZLIB_H + /* Define if you have the coug library (-lcoug). */ #undef HAVE_LIBCOUG @@ -81,3 +87,6 @@ /* Define if you have the mpio library (-lmpio). */ #undef HAVE_LIBMPIO + +/* Define if you have the z library (-lz). */ +#undef HAVE_LIBZ diff --git a/src/H5private.h b/src/H5private.h index 33493da..328f907 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -206,6 +206,19 @@ typedef struct { #endif /* + * Data types and functions for timing certain parts of the library. + */ +typedef struct { + double utime; /*user time */ + double stime; /*system time */ + double etime; /*elapsed wall-clock time */ +} H5_timer_t; + +void H5_timer_reset (H5_timer_t *timer); +void H5_timer_begin (H5_timer_t *timer); +void H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/); + +/* * Redefine all the POSIX functions. We should never see a POSIX * function (or any other non-HDF5 function) in the source! */ diff --git a/src/Makefile.in b/src/Makefile.in index 20b4ae3..ca59511 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -18,10 +18,10 @@ PARALLEL_SRC=H5Fmpio.c LIB_SRC=H5.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcore.c H5Ffamily.c \ H5Fistore.c H5Flow.c H5Fsec2.c H5Fsplit.c H5Fstdio.c H5G.c H5Gent.c \ - H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c H5Ocont.c \ - H5Odtype.c H5Oefl.c H5Olayout.c H5Oname.c H5Onull.c H5Osdspace.c \ - H5Oshared.c H5Ostab.c H5P.c H5S.c H5Ssimp.c H5T.c H5Tconv.c H5Tinit.c \ - H5V.c @PARALLEL_SRC@ + H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c H5Ocomp.c \ + H5Ocont.c H5Odtype.c H5Oefl.c H5Olayout.c H5Oname.c H5Onull.c \ + H5Osdspace.c H5Oshared.c H5Ostab.c H5P.c H5S.c H5Ssimp.c H5T.c \ + H5Tconv.c H5Tinit.c H5V.c H5Z.c @PARALLEL_SRC@ LIB_OBJ=$(LIB_SRC:.c=.o) @@ -36,13 +36,13 @@ PROG_OBJ=$(PROG_SRC:.c=.o) PUB_HDR=H5public.h H5ACpublic.h H5Bpublic.h H5Ppublic.h H5Dpublic.h \ H5Epublic.h H5Fpublic.h H5Gpublic.h H5HGpublic.h H5HLpublic.h \ H5Ipublic.h H5MFpublic.h H5MMpublic.h H5Opublic.h H5Spublic.h \ - H5Tpublic.h H5config.h hdf5.h + H5Tpublic.h H5Zpublic.h H5config.h hdf5.h # Other header files (not to be installed)... PRIVATE_HDR=H5private.h H5ACprivate.h H5Bprivate.h H5Pprivate.h H5Dprivate.h \ H5Eprivate.h H5Fprivate.h H5Gprivate.h H5Gpkg.h H5HGprivate.h \ H5HLprivate.h H5Iprivate.h H5MFprivate.h H5MMprivate.h H5Oprivate.h \ - H5Sprivate.h H5Tprivate.h H5Tpkg.h H5Vprivate.h + H5Sprivate.h H5Tprivate.h H5Tpkg.h H5Vprivate.h H5Zprivate.h # Number format detection H5Tinit.c: H5detect diff --git a/src/debug.c b/src/debug.c index f5370b4..4b8fdc3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -26,6 +26,7 @@ #define INDENT 3 #define VCOL 50 + /*------------------------------------------------------------------------- * Function: main @@ -51,7 +52,7 @@ main(int argc, char *argv[]) H5F_t *f; haddr_t addr; uint8 sig[16]; - intn i; + intn i, ndims; herr_t status = SUCCEED; haddr_t extra; @@ -123,12 +124,18 @@ main(int argc, char *argv[]) * subclass. The subclass identifier is the byte immediately * after the B-tree signature. */ - H5B_subid_t subtype = (H5B_subid_t)sig[H5B_SIZEOF_MAGIC]; + H5B_subid_t subtype = (H5B_subid_t)sig[H5B_SIZEOF_MAGIC]; + switch (subtype) { case H5B_SNODE_ID: status = H5G_node_debug(f, &addr, stdout, 0, VCOL, &extra); break; + case H5B_ISTORE_ID: + ndims = (int)extra.offset; + status = H5F_istore_debug (f, &addr, stdout, 0, VCOL, ndims); + break; + default: fprintf(stderr, "Unknown B-tree subtype %u\n", (unsigned) (subtype)); HDexit(4); @@ -44,7 +44,7 @@ list (hid_t group, const char *name, void __unused__ *op_data) hid_t (*func)(void*); void *edata; int i; - char linkval[512]; + char buf[512]; H5G_stat_t statbuf; /* Disable error reporting */ @@ -52,19 +52,20 @@ list (hid_t group, const char *name, void __unused__ *op_data) H5Eset_auto (NULL, NULL); /* Print info about each name */ - printf ("%-30s", name); + printf ("%-25s ", name); if (H5Gstat (group, name, TRUE, &statbuf)>=0) { - printf ("FILE={%lu,%lu}, OID={%lu,%lu} ", - statbuf.fileno[0], statbuf.fileno[1], - statbuf.objno[0], statbuf.objno[1]); + sprintf (buf, "%lu:%lu:%lu:%lu", + statbuf.fileno[1], statbuf.fileno[0], + statbuf.objno[1], statbuf.objno[0]); + printf ("%-20s ", buf); } if ((obj=H5Dopen (group, name))>=0) { hsize_t size[64]; hid_t space = H5Dget_space (obj); int ndims = H5Sget_dims (space, size); - printf (" Dataset {"); + printf ("Dataset {"); for (i=0; i<ndims; i++) { HDfprintf (stdout, "%s%Hu", i?", ":"", size[i]); } @@ -72,15 +73,15 @@ list (hid_t group, const char *name, void __unused__ *op_data) H5Dclose (space); H5Dclose (obj); } else if ((obj=H5Gopen (group, name))>=0) { - printf (" Group\n"); + printf ("Group\n"); H5Gclose (obj); - } else if (H5Gget_linkval (group, name, sizeof(linkval), linkval)>=0) { - if (NULL==HDmemchr (linkval, 0, sizeof(linkval))) { - strcpy (linkval+sizeof(linkval)-4, "..."); + } else if (H5Gget_linkval (group, name, sizeof(buf), buf)>=0) { + if (NULL==HDmemchr (buf, 0, sizeof(buf))) { + strcpy (buf+sizeof(buf)-4, "..."); } - printf (" -> %s\n", linkval); + printf (" -> %s\n", buf); } else { - printf (" Unknown Type\n"); + printf ("Unknown Type\n"); } /* Restore error reporting */ @@ -34,4 +34,6 @@ #include <H5Ppublic.h> /* Property lists */ #include <H5Spublic.h> /* Dataspaces */ #include <H5Tpublic.h> /* Datatypes */ +#include <H5Zpublic.h> /* Data compression */ + #endif diff --git a/test/.distdep b/test/.distdep index cbe325d..e7c7b05 100644 --- a/test/.distdep +++ b/test/.distdep @@ -3,9 +3,7 @@ testhdf5.o: \ testhdf5.h \ ../src/H5private.h \ ../src/H5public.h \ - ../src/H5config.h \ - ../src/H5Eprivate.h \ - ../src/H5Epublic.h + ../src/H5config.h tfile.o: \ tfile.c \ testhdf5.h \ @@ -19,7 +17,8 @@ tfile.o: \ ../src/H5Bpublic.h \ ../src/H5Fprivate.h \ ../src/H5Fpublic.h \ - ../src/H5Pprivate.h + ../src/H5Pprivate.h \ + ../src/H5Ppublic.h theap.o: \ theap.c \ testhdf5.h \ @@ -36,7 +35,8 @@ theap.o: \ ../src/H5Fpublic.h \ ../src/H5Pprivate.h \ ../src/H5Ppublic.h \ - ../src/H5Dpublic.h + ../src/H5Dpublic.h \ + ../src/H5Zpublic.h tmeta.o: \ tmeta.c \ testhdf5.h \ @@ -44,7 +44,8 @@ tmeta.o: \ ../src/H5public.h \ ../src/H5config.h \ ../src/H5Eprivate.h \ - ../src/H5Epublic.h + ../src/H5Epublic.h \ + ../src/H5Ipublic.h tohdr.o: \ tohdr.c \ testhdf5.h \ @@ -62,6 +63,7 @@ tohdr.o: \ ../src/H5Pprivate.h \ ../src/H5Ppublic.h \ ../src/H5Dpublic.h \ + ../src/H5Zpublic.h \ ../src/H5Gprivate.h \ ../src/H5Gpublic.h \ ../src/H5Bprivate.h \ @@ -91,6 +93,7 @@ tstab.o: \ ../src/H5Pprivate.h \ ../src/H5Ppublic.h \ ../src/H5Dpublic.h \ + ../src/H5Zpublic.h \ ../src/H5Gprivate.h \ ../src/H5Gpublic.h \ ../src/H5Bprivate.h \ @@ -125,7 +128,11 @@ th5s.o: \ ../src/H5HGprivate.h \ ../src/H5HGpublic.h \ ../src/H5Tprivate.h \ - ../src/H5Tpublic.h + ../src/H5Tpublic.h \ + ../src/H5Zprivate.h \ + ../src/H5Zpublic.h \ + ../src/H5Pprivate.h \ + ../src/H5Ppublic.h dtypes.o: \ dtypes.c \ ../src/hdf5.h \ @@ -144,6 +151,7 @@ dtypes.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ ../src/H5Spublic.h \ ../src/H5Tpublic.h \ ../src/H5Iprivate.h \ @@ -167,6 +175,7 @@ istore.o: \ ../src/H5Ppublic.h \ ../src/H5Dpublic.h \ ../src/H5Fpublic.h \ + ../src/H5Zpublic.h \ ../src/H5Fprivate.h \ ../src/H5Gprivate.h \ ../src/H5Gpublic.h \ @@ -180,8 +189,7 @@ istore.o: \ ../src/H5HGpublic.h \ ../src/H5Tprivate.h \ ../src/H5Tpublic.h \ - ../src/H5Sprivate.h \ - ../src/H5Spublic.h + ../src/H5Sprivate.h dsets.o: \ dsets.c \ ../src/hdf5.h \ @@ -200,6 +208,7 @@ dsets.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ ../src/H5Spublic.h \ ../src/H5Tpublic.h cmpd_dset.o: \ @@ -220,7 +229,9 @@ cmpd_dset.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ - ../src/H5Spublic.h + ../src/H5Zpublic.h \ + ../src/H5Spublic.h \ + ../src/H5Tpublic.h extend.o: \ extend.c \ ../src/hdf5.h \ @@ -239,7 +250,7 @@ extend.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ - ../src/H5Spublic.h + ../src/H5Zpublic.h external.o: \ external.c \ ../src/hdf5.h \ @@ -258,6 +269,7 @@ external.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ ../src/H5Spublic.h \ ../src/H5Tpublic.h iopipe.o: \ @@ -278,6 +290,7 @@ iopipe.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ ../src/H5Spublic.h \ ../src/H5Tpublic.h gheap.o: \ @@ -295,7 +308,8 @@ gheap.o: \ ../src/H5Bprivate.h \ ../src/H5Bpublic.h \ ../src/H5HGprivate.h \ - ../src/H5HGpublic.h + ../src/H5HGpublic.h \ + ../src/H5Pprivate.h shtype.o: \ shtype.c \ ../src/hdf5.h \ @@ -314,7 +328,7 @@ shtype.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ - ../src/H5Spublic.h + ../src/H5Zpublic.h big.o: \ big.c \ ../src/hdf5.h \ @@ -333,8 +347,10 @@ big.o: \ ../src/H5MMpublic.h \ ../src/H5Opublic.h \ ../src/H5Ppublic.h \ + ../src/H5Zpublic.h \ ../src/H5Spublic.h \ - ../src/H5Tpublic.h + ../src/H5Tpublic.h \ + ../src/H5private.h links.o: \ links.c \ ../src/hdf5.h \ diff --git a/test/dsets.c b/test/dsets.c index bf4b75f..fab1f75 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -35,6 +35,7 @@ #define DSET_CHUNKED_NAME "chunked" #define DSET_SIMPLE_IO_NAME "simple_io" #define DSET_TCONV_NAME "tconv" +#define DSET_COMPRESS_NAME "compressed" /*------------------------------------------------------------------------- @@ -89,6 +90,7 @@ test_create(hid_t file) void *client_data = NULL; printf("%-70s", "Testing create/open/close"); + fflush (stdout); /* Create the data space */ dims[0] = 256; @@ -202,6 +204,7 @@ test_simple_io(hid_t file) void *tconv_buf = NULL; printf("%-70s", "Testing simple I/O"); + fflush (stdout); /* Initialize the dataset */ for (i = n = 0; i < 100; i++) { @@ -290,7 +293,8 @@ test_tconv(hid_t file) assert (in); printf("%-70s", "Testing data type conversion"); - + fflush (stdout); + /* Initialize the dataset */ for (i = 0; i < 1000000; i++) { out[i*4+0] = 0x11; @@ -346,6 +350,132 @@ test_tconv(hid_t file) puts(" PASSED"); return 0; } + + +/*------------------------------------------------------------------------- + * Function: test_compression + * + * Purpose: Tests dataset compression. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Robb Matzke + * Wednesday, April 15, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +test_compression(hid_t file) +{ + hid_t dataset, space, xfer, dc; + herr_t status; + int points[100][200], check[100][200]; + int i, j, n; + hsize_t dims[2], chunk_size[2]; + void *tconv_buf = NULL; + + printf("%-70s", "Testing compression"); + fflush (stdout); + + /* Initialize the dataset */ + for (i = n = 0; i < 100; i++) { + for (j = 0; j < 100; j++) { + points[i][j] = n++; + } + } + + /* Create the data space */ + dims[0] = 100; + dims[1] = 200; + space = H5Screate_simple(2, dims, NULL); + assert(space>=0); + + /* Create a small conversion buffer to test strip mining */ + tconv_buf = malloc (1000); + xfer = H5Pcreate (H5P_DATASET_XFER); + assert (xfer>=0); + status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL); + assert (status>=0); + + /* Use chunked storage with compression */ + dc = H5Pcreate (H5P_DATASET_CREATE); + chunk_size[0] = 2; + chunk_size[1] = 25; + H5Pset_chunk (dc, 2, chunk_size); + H5Pset_deflate (dc, 6); + + /* Create the dataset */ + dataset = H5Dcreate(file, DSET_COMPRESS_NAME, H5T_NATIVE_INT, space, dc); + assert(dataset >= 0); + + /* Write the data to the dataset */ + status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, points); + if (status<0) goto error; + + /* Read the dataset back */ + status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, check); + if (status<0) goto error; + + /* Check that the values read are the same as the values written */ + for (i = 0; i < 100; i++) { + for (j = 0; j < 200; j++) { + if (points[i][j] != check[i][j]) { + puts("*FAILED*"); + printf(" Read different values than written.\n"); + printf(" At index %d,%d\n", i, j); + goto error; + } + } + } + + /* + * Write some random data to the dataset, hopefully causing chunks to be + * reallocated as they grow. + */ + for (i=0; i<100; i++) { + for (j=0; j<100; j++) { + points[i][j] = rand (); + } + } + status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, points); + if (status<0) goto error; + + + /* Read the dataset back and check it */ + status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + xfer, check); + if (status<0) goto error; + + /* Check that the values read are the same as the values written */ + for (i = 0; i < 100; i++) { + for (j = 0; j < 200; j++) { + if (points[i][j] != check[i][j]) { + puts("*FAILED*"); + printf(" Read different values than written.\n"); + printf(" At index %d,%d\n", i, j); + goto error; + } + } + } + + + + + H5Dclose(dataset); + + puts(" PASSED"); + return 0; + + error: + return -1; +} /*------------------------------------------------------------------------- * Function: main @@ -366,7 +496,7 @@ test_tconv(hid_t file) int main(void) { - hid_t file; + hid_t file, grp; herr_t status; int nerrors = 0; @@ -377,9 +507,15 @@ main(void) H5Eset_auto (display_error_cb, NULL); unlink("dataset.h5"); - file = H5Fcreate("dataset.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate("dataset.h5", H5F_ACC_TRUNC|H5F_ACC_DEBUG, + H5P_DEFAULT, H5P_DEFAULT); assert(file >= 0); + /* Cause the library to emit initial messages */ + grp = H5Gcreate (file, "emit diagnostics", 0); + H5Gclose (grp); + + status = test_create(file); nerrors += status < 0 ? 1 : 0; @@ -388,6 +524,9 @@ main(void) status = test_tconv(file); nerrors += status < 0 ? 1 : 0; + + status = test_compression(file); + nerrors += status < 0 ? 1 : 0; status = H5Fclose(file); diff --git a/test/istore.c b/test/istore.c index fa90b55..9dc4de2 100644 --- a/test/istore.c +++ b/test/istore.c @@ -316,7 +316,8 @@ test_extend(H5F_t *f, const char *prefix, memset(buf, (signed)(128+ctr), (size_t)nelmts); /* Write to disk */ - if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) { + if (H5F_arr_write(f, &layout, NULL, NULL, size, size, zero, offset, + buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); @@ -326,7 +327,7 @@ test_extend(H5F_t *f, const char *prefix, } /* Read from disk */ memset(check, 0xff, (size_t)nelmts); - if (H5F_arr_read(f, &layout, NULL, size, size, zero, offset, + if (H5F_arr_read(f, &layout, NULL, NULL, size, size, zero, offset, check)<0) { puts("*FAILED*"); if (!isatty(1)) { @@ -362,8 +363,8 @@ test_extend(H5F_t *f, const char *prefix, /* Now read the entire array back out and check it */ memset(buf, 0xff, nx * ny * nz); - if (H5F_arr_read(f, &layout, NULL, whole_size, whole_size, zero, zero, - buf)<0) { + if (H5F_arr_read(f, &layout, NULL, NULL, whole_size, whole_size, zero, + zero, buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); @@ -486,7 +487,8 @@ test_sparse(H5F_t *f, const char *prefix, size_t nblocks, memset(buf, (signed)(128+ctr), nx * ny * nz); /* write to disk */ - if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) { + if (H5F_arr_write(f, &layout, NULL, NULL, size, size, zero, offset, + buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); |