From 9bba487ca4dc89140684da78cd541aba4403df53 Mon Sep 17 00:00:00 2001 From: Robb Matzke Date: Fri, 23 Oct 1998 10:44:52 -0500 Subject: [svn-r780] *** empty log message *** --- doc/html/Chunk_f1.gif | Bin 0 -> 3664 bytes doc/html/Chunk_f1.obj | 252 +++++++++++++++++++++++++++++++++++++++++++++++++ doc/html/Chunk_f2.gif | Bin 0 -> 3986 bytes doc/html/Chunk_f2.obj | 95 +++++++++++++++++++ doc/html/Chunk_f3.gif | Bin 0 -> 6815 bytes doc/html/Chunk_f4.gif | Bin 0 -> 5772 bytes doc/html/Chunk_f5.gif | Bin 0 -> 5455 bytes doc/html/Chunk_f6.gif | Bin 0 -> 4949 bytes doc/html/Chunk_f6.obj | 107 +++++++++++++++++++++ doc/html/Chunking.html | 193 +++++++++++++++++++++++++++++++++++++ 10 files changed, 647 insertions(+) create mode 100644 doc/html/Chunk_f1.gif create mode 100644 doc/html/Chunk_f1.obj create mode 100644 doc/html/Chunk_f2.gif create mode 100644 doc/html/Chunk_f2.obj create mode 100644 doc/html/Chunk_f3.gif create mode 100644 doc/html/Chunk_f4.gif create mode 100644 doc/html/Chunk_f5.gif create mode 100644 doc/html/Chunk_f6.gif create mode 100644 doc/html/Chunk_f6.obj create mode 100644 doc/html/Chunking.html diff --git a/doc/html/Chunk_f1.gif b/doc/html/Chunk_f1.gif new file mode 100644 index 0000000..d73201a Binary files /dev/null and b/doc/html/Chunk_f1.gif differ diff --git a/doc/html/Chunk_f1.obj b/doc/html/Chunk_f1.obj new file mode 100644 index 0000000..004204a --- /dev/null +++ b/doc/html/Chunk_f1.obj @@ -0,0 +1,252 @@ +%TGIF 3.0-p17 +state(0,33,100.000,0,0,0,16,1,9,1,1,0,0,0,1,1,1,'Courier',0,17,0,0,0,10,0,0,1,1,0,16,0,0,1,1,1,0,1088,1408,0,0,2880). +% +% @(#)$Header$ +% %W% +% +unit("1 pixel/pixel"). +color_info(11,65535,0,[ + "magenta", 65535, 0, 65535, 65280, 0, 65280, 1, + "red", 65535, 0, 0, 65280, 0, 0, 1, + "green", 0, 65535, 0, 0, 65280, 0, 1, + "blue", 0, 0, 65535, 0, 0, 65280, 1, + "yellow", 65535, 65535, 0, 65280, 65280, 0, 1, + "pink", 65535, 49344, 52171, 65280, 49152, 51968, 1, + "cyan", 0, 65535, 65535, 0, 65280, 65280, 1, + "CadetBlue", 24415, 40606, 41120, 24320, 40448, 40960, 1, + "white", 65535, 65535, 65535, 65280, 65280, 65280, 1, + "black", 0, 0, 0, 0, 0, 0, 1, + "DarkSlateGray", 12079, 20303, 20303, 12032, 20224, 20224, 1 +]). +page(1,"",1). +text('black',432,272,'Courier',0,17,2,1,0,1,49,28,302,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Point", + "Written"]). +box('black',256,288,320,352,0,3,1,70,0,0,0,0,0,'3',[ +]). +text('black',288,272,'Courier',0,17,1,1,0,1,49,14,75,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Dataset"]). +box('black',352,288,384,320,5,1,1,77,5,0,0,0,0,'1',[ +]). +text('black',368,272,'Courier',0,17,1,1,0,1,35,14,80,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Chunk"]). +box('black',96,32,544,384,0,1,1,118,0,0,0,0,0,'1',[ +]). +box('black',128,64,256,128,5,1,1,131,5,0,0,0,0,'1',[ +]). +box('black',128,128,256,192,5,1,1,132,5,0,0,0,0,'1',[ +]). +box('black',384,64,512,128,5,1,1,137,5,0,0,0,0,'1',[ +]). +box('black',256,128,384,192,5,1,1,142,5,0,0,0,0,'1',[ +]). +box('black',256,192,384,256,5,1,1,144,5,0,0,0,0,'1',[ +]). +box('black',384,192,512,256,5,1,1,146,5,0,0,0,0,'1',[ +]). +box('black',128,64,432,224,0,3,1,26,0,0,0,0,0,'3',[ +]). +group([ +polygon('black',11,[ + 152,80,154,86,160,86,155,89,157,94,152,91,147,94,149,89, + 144,86,150,86,152,80],1,1,1,0,178,0,0,0,0,0,'1', + "000",[ +]), +box('black',148,84,156,92,0,1,0,179,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',152,83,'Courier',0,17,1,1,0,1,112,14,180,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',152,80,'Courier',0,17,1,1,0,1,0,14,181,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +182,0,0,[ +]). +group([ +polygon('black',11,[ + 200,96,202,102,208,102,203,105,205,110,200,107,195,110,197,105, + 192,102,198,102,200,96],1,1,1,0,188,0,0,0,0,0,'1', + "000",[ +]), +box('black',196,100,204,108,0,1,0,189,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',200,99,'Courier',0,17,1,1,0,1,112,14,190,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',200,96,'Courier',0,17,1,1,0,1,0,14,191,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +192,0,0,[ +]). +group([ +polygon('black',11,[ + 168,128,170,134,176,134,171,137,173,142,168,139,163,142,165,137, + 160,134,166,134,168,128],1,1,1,0,198,0,0,0,0,0,'1', + "000",[ +]), +box('black',164,132,172,140,0,1,0,199,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',168,131,'Courier',0,17,1,1,0,1,112,14,200,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',168,128,'Courier',0,17,1,1,0,1,0,14,201,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +202,0,0,[ +]). +group([ +polygon('black',11,[ + 168,160,170,166,176,166,171,169,173,174,168,171,163,174,165,169, + 160,166,166,166,168,160],1,1,1,0,208,0,0,0,0,0,'1', + "000",[ +]), +box('black',164,164,172,172,0,1,0,209,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',168,163,'Courier',0,17,1,1,0,1,112,14,210,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',168,160,'Courier',0,17,1,1,0,1,0,14,211,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +212,0,0,[ +]). +group([ +polygon('black',11,[ + 136,144,138,150,144,150,139,153,141,158,136,155,131,158,133,153, + 128,150,134,150,136,144],1,1,1,0,218,0,0,0,0,0,'1', + "000",[ +]), +box('black',132,148,140,156,0,1,0,219,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',136,147,'Courier',0,17,1,1,0,1,112,14,220,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',136,144,'Courier',0,17,1,1,0,1,0,14,221,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +222,0,0,[ +]). +group([ +polygon('black',11,[ + 248,144,250,150,256,150,251,153,253,158,248,155,243,158,245,153, + 240,150,246,150,248,144],1,1,1,0,228,0,0,0,0,0,'1', + "000",[ +]), +box('black',244,148,252,156,0,1,0,229,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',248,147,'Courier',0,17,1,1,0,1,112,14,230,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',248,144,'Courier',0,17,1,1,0,1,0,14,231,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +232,0,0,[ +]). +group([ +polygon('black',11,[ + 296,176,298,182,304,182,299,185,301,190,296,187,291,190,293,185, + 288,182,294,182,296,176],1,1,1,0,238,0,0,0,0,0,'1', + "000",[ +]), +box('black',292,180,300,188,0,1,0,239,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',296,179,'Courier',0,17,1,1,0,1,112,14,240,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',296,176,'Courier',0,17,1,1,0,1,0,14,241,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +242,0,0,[ +]). +group([ +polygon('black',11,[ + 360,208,362,214,368,214,363,217,365,222,360,219,355,222,357,217, + 352,214,358,214,360,208],1,1,1,0,248,0,0,0,0,0,'1', + "000",[ +]), +box('black',356,212,364,220,0,1,0,249,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',360,211,'Courier',0,17,1,1,0,1,112,14,250,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',360,208,'Courier',0,17,1,1,0,1,0,14,251,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +252,0,0,[ +]). +group([ +polygon('black',11,[ + 408,192,410,198,416,198,411,201,413,206,408,203,403,206,405,201, + 400,198,406,198,408,192],1,1,1,0,258,0,0,0,0,0,'1', + "000",[ +]), +box('black',404,196,412,204,0,1,0,259,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',408,195,'Courier',0,17,1,1,0,1,112,14,260,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',408,192,'Courier',0,17,1,1,0,1,0,14,261,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +262,0,0,[ +]). +group([ +polygon('black',11,[ + 376,128,378,134,384,134,379,137,381,142,376,139,371,142,373,137, + 368,134,374,134,376,128],1,1,1,0,268,0,0,0,0,0,'1', + "000",[ +]), +box('black',372,132,380,140,0,1,0,269,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',376,131,'Courier',0,17,1,1,0,1,112,14,270,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',376,128,'Courier',0,17,1,1,0,1,0,14,271,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +272,0,0,[ +]). +group([ +polygon('black',11,[ + 408,80,410,86,416,86,411,89,413,94,408,91,403,94,405,89, + 400,86,406,86,408,80],1,1,1,0,278,0,0,0,0,0,'1', + "000",[ +]), +box('black',404,84,412,92,0,1,0,279,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',408,83,'Courier',0,17,1,1,0,1,112,14,280,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',408,80,'Courier',0,17,1,1,0,1,0,14,281,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +282,0,0,[ +]). +group([ +polygon('black',11,[ + 424,304,426,310,432,310,427,313,429,318,424,315,419,318,421,313, + 416,310,422,310,424,304],1,1,1,0,288,0,0,0,0,0,'1', + "000",[ +]), +box('black',420,308,428,316,0,1,0,289,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',424,307,'Courier',0,17,1,1,0,1,112,14,290,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "", 1, 0, 0, +text('black',424,304,'Courier',0,17,1,1,0,1,0,14,291,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + ""])) +]) +], +292,0,0,[ +]). diff --git a/doc/html/Chunk_f2.gif b/doc/html/Chunk_f2.gif new file mode 100644 index 0000000..68f9433 Binary files /dev/null and b/doc/html/Chunk_f2.gif differ diff --git a/doc/html/Chunk_f2.obj b/doc/html/Chunk_f2.obj new file mode 100644 index 0000000..7361c1c --- /dev/null +++ b/doc/html/Chunk_f2.obj @@ -0,0 +1,95 @@ +%TGIF 3.0-p17 +state(0,33,100.000,0,0,0,16,1,9,1,1,6,1,1,0,1,0,'Courier',0,17,0,2,0,10,0,0,1,1,0,16,0,0,1,1,1,0,1088,1408,0,0,2880). +% +% @(#)$Header$ +% %W% +% +unit("1 pixel/pixel"). +color_info(11,65535,0,[ + "magenta", 65535, 0, 65535, 65280, 0, 65280, 1, + "red", 65535, 0, 0, 65280, 0, 0, 1, + "green", 0, 65535, 0, 0, 65280, 0, 1, + "blue", 0, 0, 65535, 0, 0, 65280, 1, + "yellow", 65535, 65535, 0, 65280, 65280, 0, 1, + "pink", 65535, 49344, 52171, 65280, 49152, 51968, 1, + "cyan", 0, 65535, 65535, 0, 65280, 65280, 1, + "CadetBlue", 24415, 40606, 41120, 24320, 40448, 40960, 1, + "white", 65535, 65535, 65535, 65280, 65280, 65280, 1, + "black", 0, 0, 0, 0, 0, 0, 1, + "DarkSlateGray", 12079, 20303, 20303, 12032, 20224, 20224, 1 +]). +page(1,"",1). +group([ +box('black',192,416,512,544,0,1,0,22,0,0,0,0,0,'1',[ +]), +oval('black',192,384,512,448,0,1,1,23,0,0,0,0,0,'1',[ +]), +arc('black',0,1,1,0,192,512,352,544,192,544,512,544,0,320,64,11520,11520,24,0,0,8,3,0,0,0,'1','8','3',[ +]), +poly('black',2,[ + 192,416,192,544],0,1,1,25,0,0,0,0,8,3,0,0,0,'1','8','3', + "0",[ +]), +poly('black',2,[ + 512,416,512,544],0,1,1,26,0,0,0,0,8,3,0,0,0,'1','8','3', + "0",[ +]), +box('black',196,452,508,572,0,1,0,27,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',352,451,'Courier',0,17,1,1,0,1,112,14,28,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "HDF5 File", 1, 0, 0, +text('black',351,505,'Courier',0,17,1,1,0,1,63,14,29,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "HDF5 File"])) +]) +], +30,0,0,[ +]). +group([ +polygon('black',5,[ + 240,160,240,352,464,352,464,160,240,160],0,1,1,0,63,0,0,0,0,0,'1', + "00",[ +]), +box('black',254,164,450,348,0,1,0,64,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',352,163,'Courier',0,17,1,1,0,1,112,14,65,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "Filter", 1, 0, 0, +text('black',351,242,'Courier',0,17,2,1,0,1,49,28,66,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Filter", + "Pipeine"])) +]) +], +62,0,0,[ +]). +group([ +polygon('black',13,[ + 304,85,304,107,336,107,336,128,368,128,368,107,400,107,400,85, + 368,85,368,64,336,64,336,85,304,85],0,1,1,0,103,0,0,0,0,0,'1', + "0000",[ +]), +box('black',307,68,397,124,0,1,0,104,0,0,0,0,0,'1',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',352,67,'Courier',0,17,1,1,0,1,112,14,105,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "Modify Bytes", 1, 0, 0, +text('black',352,89,'Courier',0,17,1,1,0,1,84,14,106,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Modify Bytes"])) +]) +], +107,0,0,[ +]). +box('black',176,48,528,592,0,1,1,143,0,0,0,0,0,'1',[ +]). +poly('black',4,[ + 256,416,256,128,256,96,304,96],1,7,1,168,1,0,2,0,22,9,0,0,0,'7','22','9', + "6",[ +]). +poly('black',4,[ + 400,96,448,96,448,128,448,416],1,7,1,173,1,0,2,0,22,9,0,0,0,'7','22','9', + "6",[ +]). +text('black',432,128,'Courier',0,17,1,0,0,1,35,14,312,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Chunk"]). +text('black',240,368,'Courier',0,17,1,0,0,1,35,14,314,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Chunk"]). diff --git a/doc/html/Chunk_f3.gif b/doc/html/Chunk_f3.gif new file mode 100644 index 0000000..e6e8457 Binary files /dev/null and b/doc/html/Chunk_f3.gif differ diff --git a/doc/html/Chunk_f4.gif b/doc/html/Chunk_f4.gif new file mode 100644 index 0000000..76f0994 Binary files /dev/null and b/doc/html/Chunk_f4.gif differ diff --git a/doc/html/Chunk_f5.gif b/doc/html/Chunk_f5.gif new file mode 100644 index 0000000..3b12174 Binary files /dev/null and b/doc/html/Chunk_f5.gif differ diff --git a/doc/html/Chunk_f6.gif b/doc/html/Chunk_f6.gif new file mode 100644 index 0000000..616946d Binary files /dev/null and b/doc/html/Chunk_f6.gif differ diff --git a/doc/html/Chunk_f6.obj b/doc/html/Chunk_f6.obj new file mode 100644 index 0000000..2b2f371 --- /dev/null +++ b/doc/html/Chunk_f6.obj @@ -0,0 +1,107 @@ +%TGIF 3.0-p17 +state(0,33,100.000,0,0,0,8,1,9,1,1,0,1,1,0,1,1,'Courier',0,17,0,0,0,10,0,0,1,1,0,16,0,0,1,1,1,0,1088,1408,0,0,2880). +% +% @(#)$Header$ +% %W% +% +unit("1 pixel/pixel"). +color_info(11,65535,0,[ + "magenta", 65535, 0, 65535, 65280, 0, 65280, 1, + "red", 65535, 0, 0, 65280, 0, 0, 1, + "green", 0, 65535, 0, 0, 65280, 0, 1, + "blue", 0, 0, 65535, 0, 0, 65280, 1, + "yellow", 65535, 65535, 0, 65280, 65280, 0, 1, + "pink", 65535, 49344, 52171, 65280, 49152, 51968, 1, + "cyan", 0, 65535, 65535, 0, 65280, 65280, 1, + "CadetBlue", 24415, 40606, 41120, 24320, 40448, 40960, 1, + "white", 65535, 65535, 65535, 65280, 65280, 65280, 1, + "black", 0, 0, 0, 0, 0, 0, 1, + "DarkSlateGray", 12079, 20303, 20303, 12032, 20224, 20224, 1 +]). +page(1,"",1). +polygon('black',5,[ + 128,256,256,256,256,320,128,320,128,256],5,1,1,0,26,5,0,0,0,0,'1', + "00",[ +]). +polygon('black',7,[ + 256,128,256,256,128,256,128,192,192,192,192,128,256,128],5,1,1,0,25,5,0,0,0,0,'1', + "00",[ +]). +polygon('black',7,[ + 128,64,256,64,256,128,192,128,192,192,128,192,128,64],5,1,1,0,24,5,0,0,0,0,'1', + "00",[ +]). +box('black',128,64,256,320,0,3,1,22,0,0,0,0,0,'3',[ +]). +text('black',192,96,'Courier',0,17,1,1,0,1,49,14,34,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 1"]). +text('black',224,160,'Courier',0,17,1,1,0,1,49,14,40,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 2"]). +text('black',192,272,'Courier',0,17,1,1,0,1,49,14,46,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 3"]). +polygon('black',5,[ + 448,256,576,256,576,320,448,320,448,256],5,1,1,0,59,5,0,0,0,0,'1', + "00",[ +]). +polygon('black',7,[ + 576,128,576,256,448,256,448,192,512,192,512,128,576,128],5,1,1,0,60,5,0,0,0,0,'1', + "00",[ +]). +polygon('black',7,[ + 448,64,576,64,576,128,512,128,512,192,448,192,448,64],5,1,1,0,61,5,0,0,0,0,'1', + "00",[ +]). +box('black',448,64,576,320,0,3,1,62,0,0,0,0,0,'3',[ +]). +text('black',512,96,'Courier',0,17,1,1,0,1,49,14,63,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 1"]). +text('black',544,160,'Courier',0,17,1,1,0,1,49,14,64,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 2"]). +text('black',512,272,'Courier',0,17,1,1,0,1,49,14,65,0,11,3,2,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "Strip 3"]). +text('black',192,32,'Courier',0,17,1,1,0,1,28,14,68,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "FILE"]). +text('black',512,32,'Courier',0,17,1,1,0,1,42,14,70,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "MEMORY"]). +group([ +polygon('black',6,[ + 320,160,320,208,384,208,416,184,384,160,320,160],0,3,1,0,72,0,0,0,0,0,'3', + "00",[ +]), +box('black',324,164,388,204,0,3,0,73,0,0,0,0,0,'3',[ +attr("", "auto_center_attr", 0, 1, 0, +text('black',356,162,'Courier',0,17,1,1,0,1,112,14,74,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "auto_center_attr"])), +attr("label=", "TCONV", 1, 0, 0, +text('black',355,177,'Courier',0,17,1,1,0,1,35,14,75,0,11,3,0,0,0,0,0,2,0,0,0,0,"",0,0,0,[ + "TCONV"])) +]) +], +76,0,0,[ +]). +poly('black',5,[ + 256,96,288,96,320,96,320,128,320,160],1,7,1,87,1,0,5,0,22,9,0,0,0,'7','22','9', + "70",[ +]). +poly('black',2,[ + 256,184,320,184],1,7,1,88,1,0,5,0,22,9,0,0,0,'7','22','9', + "0",[ +]). +poly('black',5,[ + 256,288,288,288,320,288,320,256,320,208],1,7,1,89,1,0,5,0,22,9,0,0,0,'7','22','9', + "70",[ +]). +poly('black',5,[ + 400,160,400,128,400,96,432,96,448,96],1,7,1,92,1,0,5,0,22,9,0,0,0,'7','22','9', + "70",[ +]). +poly('black',2,[ + 416,184,512,184],1,7,1,93,1,0,5,0,22,9,0,0,0,'7','22','9', + "0",[ +]). +poly('black',5,[ + 400,208,400,256,400,288,432,288,448,288],1,7,1,94,1,0,5,0,22,9,0,0,0,'7','22','9', + "70",[ +]). +box('black',96,0,608,352,0,1,1,99,0,0,0,0,0,'1',[ +]). diff --git a/doc/html/Chunking.html b/doc/html/Chunking.html new file mode 100644 index 0000000..ccf4f9e --- /dev/null +++ b/doc/html/Chunking.html @@ -0,0 +1,193 @@ + + + + Dataset Chunking Pitfalls + + + +

Dataset Chunking Pitfalls

+ +

Table of Contents

+ + + +

1. Introduction

+ + +

Chunking refers to a storage layout where a dataset is + partitioned into fixed-size multi-dimensional chunks. The + chunks cover the dataset but the dataset need not be an integral + number of chunks. If no data is ever written to a chunk then + that chunk isn't allocated on disk. Figure 1 shows a 25x48 + element dataset covered by nine 10x20 chunks and 11 data points + written to the dataset. No data was written to the region of + the dataset covered by three of the chunks so those chunks were + never allocated in the file -- the other chunks are allocated at + independent locations in the file and written in their entirety. + +


Figure 1
+ +

The HDF5 library treats chunks as atomic objects -- disk I/O is + always in terms of complete chunks(1). This + allows data filters to be defined by the application to perform + tasks such as compression, encryption, checksumming, + etc. on entire chunks. As shown in Figure 2, if + H5Dwrite() touches only a few bytes of the chunk, + the entire chunk is read from the file, the data passes upward + through the filter pipeline, the few bytes are modified, the + data passes downward through the filter pipeline, and the entire + chunk is written back to the file. + +


Figure 2
+ +

2. The Raw Data Chunk Cache

+ +

It's obvious from Figure 2 that calling H5Dwrite() + many times from the application would result in poor performance + even if the data being written all falls within a single chunk. + A raw data chunk cache layer was added between the top of the + filter stack and the bottom of the byte modification layer(2). By default, the chunk cache will store 521 + chunks or 1MB of data (whichever is less) but these values can + be modified with H5Pset_cache(). + +

The preemption policy for the cache favors certain chunks and + tries not to preempt them. + +

+ +

3. Cache Efficiency

+ +

Now for some real numbers... A 2000x2000 element dataset is + created and covered by a 20x20 array of chunks (each chunk is 100x100 + elements). The raw data cache is adjusted to hold at most 25 chunks by + setting the maximum number of bytes to 25 times the chunk size in + bytes. Then the application creates a square, two-dimensional memory + buffer and uses it as a window into the dataset, first reading and then + rewriting in row-major order by moving the window across the dataset + (the read and write tests both start with a cold cache). + +

The measure of efficiency in Figure 3 is the number of bytes requested + by the application divided by the number of bytes transferred from the + file. There are at least a couple ways to get an estimate of the cache + performance: one way is to turn on cache + debugging and look at the number of cache misses. A more accurate + and specific way is to register a data filter whose sole purpose is to + count the number of bytes that pass through it (that's the method used + below). + +


Figure 3
+ +

The read efficiency is less than one for two reasons: collisions in the + cache are handled by preempting one of the colliding chunks, and the + preemption algorithm occasionally preempts a chunk which hasn't been + referenced for a long time but is about to be referenced in the near + future. + +

The write test results in lower efficiency for most window + sizes because HDF5 is unaware that the application is about to + overwrite the entire dataset and must read in most chunks before + modifying parts of them. + +

There is a simple way to improve efficiency for this example. + It turns out that any chunk that has been completely read or + written is a good candidate for preemption. If we increase the + penalty for such chunks from the default 0.75 to the maximum + 1.00 then efficiency improves. + +


Figure 4
+ +

The read efficiency is still less than one because of + collisions in the cache. The number of collisions can often be + reduced by increasing the number of slots in the cache. Figure + 5 shows what happens when the maximum number of slots is + increased by an order of magnitude from the default (this change + has no major effect on memory used by the test since the byte + limit was not increased for the cache). + +


Figure 5
+ +

Although the application eventually overwrites every chunk + completely the library has know way of knowing this before hand + since most calls to H5Dwrite() modify only a + portion of any given chunk. Therefore, the first modification of + a chunk will cause the chunk to be read from disk into the chunk + buffer through the filter pipeline. Eventually HDF5 might + contain a data set transfer property that can turn off this read + operation resulting in write efficiency which is equal to read + efficiency. + + +

4. Fragmentation

+ +

Even if the application transfers the entire dataset contents with a + single call to H5Dread() or H5Dwrite() it's + possible the request will be broken into smaller, more manageable + pieces by the library. This is almost certainly true if the data + transfer includes a type conversion. + +


Figure 6
+ +

By default the strip size is 1MB but it can be changed by calling + H5Pset_buffer(). + + +

5. File Storage Overhead

+ +

The chunks of the dataset are allocated at independent + locations throughout the HDF5 file and a B-tree maps chunk + N-dimensional addresses to file addresses. The more chunks that + are allocated for a dataset the larger the B-tree. Large B-trees + have two disadvantages: + +

+ +

There are three ways to reduce the number of B-tree nodes. The + obvious way is to reduce the number of chunks by choosing a larger chunk + size (doubling the chunk size will cut the number of B-tree nodes in + half). Another method is to adjust the split ratios for the B-tree by + calling H5Pset_split_ratios(), but this method typically + results in only a slight improvement over the default settings. + Finally, the out-degree of each node can be increased by calling + H5Pset_istore_k() (increasing the out degree actually + increases file overhead while decreasing the number of nodes). + +


+ +

Footnote 1: Parallel versions of the library + can access individual bytes of a chunk when the underlying file + uses MPI-IO. + +

Footnote 2: The raw data chunk cache was + added before the second alpha release. + + + +


+
Robb Matzke
+ + +Last modified: Fri Oct 23 10:30:52 EDT 1998 + + + -- cgit v0.12