merge trunk

4 files changed, 243 insertions, 36 deletions

diff --git a/doc/TclZlib.3 b/doc/TclZlib.3
index 1b5e892..ebd294b 100644
--- a/doc/TclZlib.3
+++ b/doc/TclZlib.3
@@ -49,9 +49,11 @@ int
 .sp
 int
 \fBTcl_ZlibStreamGet\fR(\fIzshandle, dataObj, count\fR)
+.sp
+\fBTcl_ZlibStreamSetCompressionDictionary\fR(\fIzshandle, compDict\fR)
 .fi
 .SH ARGUMENTS
-.AS Tcl_ZlibStream *zshandlePtr out
+.AS Tcl_ZlibStream zshandle in
 .AP Tcl_Interp *interp in
 The interpreter to store resulting compressed or uncompressed data in. Also
 where any error messages are written. For \fBTcl_ZlibStreamInit\fR, this can
@@ -108,6 +110,13 @@ trailer demanded by the format is written.
 .AP int count in
 The maximum number of bytes to get from the stream, or -1 to get all remaining
 bytes from the stream's buffers.
+.AP Tcl_Obj *compDict in
+A byte array object that is the compression dictionary to use with the stream.
+Note that this is \fInot a Tcl dictionary\fR, and it is recommended that this
+only ever be used with streams that were created with their \fIformat\fR set
+to \fBTCL_ZLIB_FORMAT_ZLIB\fR because the other formats have no mechanism to
+indicate whether a compression dictionary was present other than to fail on
+decompression.
 .BE
 .SH DESCRIPTION
 These functions form the interface from the Tcl library to the Zlib
@@ -172,6 +181,25 @@ uncompressed data according to the format, and \fBTcl_ZlibStreamEof\fR returns
 a boolean value indicating whether the end of the uncompressed data has been
 reached.
 .PP
+\fBTcl_ZlibStreamSetCompressionDictionary\fR is used to control the
+compression dictionary used with the stream, a compression dictionary being an
+array of bytes (such as might be created with \fBTcl_NewByteArrayObj\fR) that
+is used to initialize the compression engine rather than leaving it to create
+it on the fly from the data being compressed. Setting a compression dictionary
+allows for more efficient compression in the case where the start of the data
+is highly regular, but it does require both the compressor and the
+decompressor to agreee on the value to use. Compression dictionaries are only
+fully supported for zlib-format data; on compression, they must be set before
+any data is sent in with \fBTcl_ZlibStreamPut\fR, and on decompression they
+should be set when \fBTcl_ZlibStreamGet\fR produces an \fBerror\fR with its
+\fB\-errorcode\fR set to
+.QW "\fBZLIB NEED_DICT\fI code\fR" ;
+the \fIcode\fR will be the Adler-32 checksum (see \fBTcl_ZlibAdler32\fR) of
+the compression dictionary sought. (Note that this is only true for
+zlib-format streams; gzip streams ignore compression dictionaries as the
+format specification doesn't permit them, and raw streams just produce a data
+error if the compression dictionary is missing or incorrect.)
+.PP
 If you wish to clear a stream and reuse it for a new compression or
 decompression action, \fBTcl_ZlibStreamReset\fR will do this and return a
 normal Tcl result code to indicate whether it was successful; if the stream is
diff --git a/doc/dict.n b/doc/dict.n
index 361a112..3bd5530 100644
--- a/doc/dict.n
+++ b/doc/dict.n
@@ -147,6 +147,31 @@ keys are treated as if they map to an empty list, and it is legal for
 there to be no items to append to the list. It is an error for the
 value that the key maps to to not be representable as a list.
 .TP
+\fBdict map \fR{\fIkeyVar valueVar\fR} \fIdictionaryValue body\fR
+.
+This command applies a transformation to each element of a dictionary,
+returning a new dictionary. It takes three arguments: the first is a
+two-element list of variable names (for the key and value respectively of each
+mapping in the dictionary), the second the dictionary value to iterate across,
+and the third a script to be evaluated for each mapping with the key and value
+variables set appropriately (in the manner of \fBlmap\fR). In an iteration
+where the evaluated script completes normally (\fBTCL_OK\fR, as opposed to an
+\fBerror\fR, etc.) the result of the script is put into an accumulator
+dictionary using the key that is the current contents of the \fIkeyVar\fR
+variable at that point. The result of the \fBdict map\fB command is the
+accumulator dictionary after all keys have been iterated over.
+.RS
+.PP
+If the evaluation of the body for any particular step generates a \fBbreak\fR,
+no further pairs from the dictionary will be iterated over and the \fBdict
+map\fR command will terminate successfully immediately. If the evaluation of
+the body for a particular step generates a \fBcontinue\fR result, the current
+iteration is aborted and the accumulator dictionary is not modified. The order
+of iteration is the natural order of the dictionary (typically the order in
+which the keys were added to the dictionary; the order is the same as that
+used in \fBdict for\fR).
+.RE
+.TP
 \fBdict merge \fR?\fIdictionaryValue ...\fR?
 .
 Return a dictionary that contains the contents of each of the
@@ -408,9 +433,9 @@ puts $foo
 #    prints: \fIa b foo {a b} bar 2 baz 3\fR
 .CE
 .SH "SEE ALSO"
-append(n), array(n), foreach(n), incr(n), list(n), lappend(n), set(n)
+append(n), array(n), foreach(n), mapeach(n), incr(n), list(n), lappend(n), set(n)
 .SH KEYWORDS
-dictionary, create, update, lookup, iterate, filter
+dictionary, create, update, lookup, iterate, filter, map
 '\" Local Variables:
 '\" mode: nroff
 '\" End:
diff --git a/doc/lmap.n b/doc/lmap.n
new file mode 100644
index 0000000..880b05a
--- /dev/null
+++ b/doc/lmap.n
@@ -0,0 +1,85 @@
+'\"
+'\" Copyright (c) 2012 Trevor Davel
+'\"
+'\" See the file "license.terms" for information on usage and redistribution
+'\" of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+'\" 
+.so man.macros
+.TH lmap n "" Tcl "Tcl Built-In Commands"
+.BS
+'\" Note:  do not modify the .SH NAME line immediately below!
+.SH NAME
+lmap \- Iterate over all elements in one or more lists and collect results
+.SH SYNOPSIS
+\fBlmap \fIvarname list body\fR
+.br
+\fBlmap \fIvarlist1 list1\fR ?\fIvarlist2 list2 ...\fR? \fIbody\fR
+.BE
+.SH DESCRIPTION
+.PP
+The \fBlmap\fR command implements a loop where the loop variable(s) take on
+values from one or more lists, and the loop returns a list of results
+collected from each iteration.
+.PP
+In the simplest case there is one loop variable, \fIvarname\fR, and one list,
+\fIlist\fR, that is a list of values to assign to \fIvarname\fR. The
+\fIbody\fR argument is a Tcl script. For each element of \fIlist\fR (in order
+from first to last), \fBlmap\fR assigns the contents of the element to
+\fIvarname\fR as if the \fBlindex\fR command had been used to extract the
+element, then calls the Tcl interpreter to execute \fIbody\fR. If execution of
+the body completes normally then the result of the body is appended to an
+accumulator list. \fBlmap\fR returns the accumulator list.
+.PP
+In the general case there can be more than one value list (e.g., \fIlist1\fR
+and \fIlist2\fR), and each value list can be associated with a list of loop
+variables (e.g., \fIvarlist1\fR and \fIvarlist2\fR). During each iteration of
+the loop the variables of each \fIvarlist\fR are assigned consecutive values
+from the corresponding \fIlist\fR. Values in each \fIlist\fR are used in order
+from first to last, and each value is used exactly once. The total number of
+loop iterations is large enough to use up all the values from all the value
+lists. If a value list does not contain enough elements for each of its loop
+variables in each iteration, empty values are used for the missing elements.
+.PP
+The \fBbreak\fR and \fBcontinue\fR statements may be invoked inside
+\fIbody\fR, with the same effect as in the \fBfor\fR and \fBforeach\fR
+commands. In these cases the body does not complete normally and the result is
+not appended to the accumulator list.
+.SH EXAMPLES
+.PP
+Zip lists together:
+.PP
+.CS
+set list1 {a b c d}
+set list2 {1 2 3 4}
+set zipped [\fBlmap\fR a $list1 b $list2 {list $a $b}] 
+# The value of zipped is "{a 1} {b 2} {c 3} {d 4}"
+.CE
+.PP
+Filter a list to remove odd values:
+.PP
+.CS
+set values {1 2 3 4 5 6 7 8}
+proc isEven {n} {expr {($n % 2) == 0}}
+set goodOnes [\fBlmap\fR x $values {expr {
+    [isEven $x] ? $x : [continue]
+}}]
+# The value of goodOnes is "2 4 6 8"
+.CE
+.PP
+Take a prefix from a list based on the contents of the list:
+.PP
+.CS
+set values {8 7 6 5 4 3 2 1}
+proc isGood {counter} {expr {$n > 3}}
+set prefix [\fBlmap\fR x $values {expr {
+    [isGood $x] ? $x : [break]
+}}]
+# The value of prefix is "8 7 6 5 4"
+.CE
+.SH "SEE ALSO"
+break(n), continue(n), for(n), foreach(n), while(n)
+.SH KEYWORDS
+foreach, iteration, list, loop, map
+'\" Local Variables:
+'\" mode: nroff
+'\" End:
diff --git a/doc/zlib.n b/doc/zlib.n
index a4ff7f8..2610527 100644
--- a/doc/zlib.n
+++ b/doc/zlib.n
@@ -1,5 +1,5 @@
 '\"
-'\" Copyright (c) 2008 Donal K. Fellows
+'\" Copyright (c) 2008-2012 Donal K. Fellows
 '\"
 '\" See the file "license.terms" for information on usage and redistribution
 '\" of this file, and for a DISCLAIMER OF ALL WARRANTIES.
@@ -170,6 +170,13 @@ the
 .QW "\fIoptions ...\fR"
 to the \fBzlib push\fR command:
 .TP
+\fB\-dictionary\fI binData\fR
+.VS "TIP 400"
+Sets the compression dictionary to use when working with compressing or
+decompressing the data to be \fIbinData\fR. Not valid for transformations that
+work with gzip-format data.
+.VE
+.TP
 \fB\-header\fI dictionary\fR
 .
 Passes a description of the gzip header to create, in the same format that
@@ -179,15 +186,33 @@ Passes a description of the gzip header to create, in the same format that
 .
 How hard to compress the data. Must be an integer from 0 (uncompressed) to 9
 (maximally compressed).
-'\".TP
-'\"\fB\-limit\fI readaheadLimit\fR
-'\".
-'\"The maximum number of bytes ahead to read.
-'\"\fITODO: not yet implemented!\fR
+.TP
+\fB\-limit\fI readaheadLimit\fR
+.
+The maximum number of bytes ahead to read when decompressing. This defaults to
+1, which ensures that data is always decompressed correctly, but may be
+increased to improve performance. This is more useful when the channel is
+non-blocking.
 .PP
 Both compressing and decompressing channel transformations add extra
-configuration options that may be accessed through \fBchan configure\fR. Each
-option is either a read-only or a write-only option. The options are:
+configuration options that may be accessed through \fBchan configure\fR. The
+options are:
+.TP
+\fB\-checksum\fI checksum\fR
+.
+This read-only option gets the current checksum for the uncompressed data that
+the compression engine has seen so far. It is valid for both compressing and
+decompressing transforms, but not for the raw inflate and deflate formats. The
+compression algorithm depends on what format is being produced or consumed.
+.TP
+\fB\-dictionary\fI binData\fR
+.VS "TIP 400"
+This read-write options gets or sets the compression dictionary to use when
+working with compressing or decompressing the data to be \fIbinData\fR. It is
+not valid for transformations that work with gzip-format data, and should not
+normally be set on compressing transformations other than at the point where
+the transformation is stacked.
+.VE
 .TP
 \fB\-flush\fI type\fR
 .
@@ -198,63 +223,80 @@ expensive flush respectively. Flushing degrades the compression ratio, but
 makes it easier for a decompressor to recover more of the file in the case of
 data corruption.
 .TP
-\fB\-checksum\fR
-.
-This read-only option gets the current checksum for the uncompressed data
-that the compression engine has seen so far. It is valid for both
-compressing and decompressing transforms, but not for the raw inflate
-and deflate formats. The compression algorithm depends on what
-format is being produced or consumed.
-.TP
-\fB\-header\fR
+\fB\-header\fI dictionary\fR
 .
 This read-only option, only valid for decompressing transforms that are
 processing gzip-format data, returns the dictionary describing the header read
 off the data stream.
+.TP
+\fB\-limit\fI readaheadLimit\fR
+.
+This read-write option is used by decompressing channels to control the
+maximum number of bytes ahead to read from the underlying data source. This
+defaults to 1, which ensures that data is always decompressed correctly, but
+may be increased to improve performance. This is more useful when the channel
+is non-blocking.
 .RE
 .SS "STREAMING SUBCOMMAND"
 .TP
-\fBzlib stream\fI mode\fR ?\fIlevel\fR?
+\fBzlib stream\fI mode\fR ?\fIoptions\fR?
 .
 Creates a streaming compression or decompression command based on the
 \fImode\fR, and return the name of the command. For a description of how that
 command works, see \fBSTREAMING INSTANCE COMMAND\fR below. The following modes
-are supported:
+and \fIoptions\fR are supported:
 .RS
 .TP
-\fBzlib stream compress\fR ?\fIlevel\fR?
+\fBzlib stream compress\fR ?\fB\-dictionary \fIbindata\fR? ?\fB\-level \fIlevel\fR?
 .
 The stream will be a compressing stream that produces zlib-format output,
 using compression level \fIlevel\fR (if specified) which will be an integer
-from 0 to 9.
+from 0 to 9,
+.VS "TIP 400"
+and the compression dictionary \fIbindata\fR (if specified).
+.VE
 .TP
-\fBzlib stream decompress\fR
+\fBzlib stream decompress\fR ?\fB\-dictionary \fIbindata\fR?
 .
 The stream will be a decompressing stream that takes zlib-format input and
 produces uncompressed output.
+.VS "TIP 400"
+If \fIbindata\fR is supplied, it is a compression dictionary to use if
+required.
+.VE
 .TP
-\fBzlib stream deflate\fR ?\fIlevel\fR?
+\fBzlib stream deflate\fR ?\fB\-dictionary \fIbindata\fR? ?\fB\-level \fIlevel\fR?
 .
 The stream will be a compressing stream that produces raw output, using
 compression level \fIlevel\fR (if specified) which will be an integer from 0
-to 9.
+to 9,
+.VS "TIP 400"
+and the compression dictionary \fIbindata\fR (if specified). Note that
+the raw compressed data includes no metadata about what compression
+dictionary was used, if any; that is a feature of the zlib-format data.
+.VE
 .TP
 \fBzlib stream gunzip\fR
 .
 The stream will be a decompressing stream that takes gzip-format input and
 produces uncompressed output.
 .TP
-\fBzlib stream gzip\fR ?\fIlevel\fR?
+\fBzlib stream gzip\fR ?\fB\-header \fIheader\fR? ?\fB\-level \fIlevel\fR?
 .
 The stream will be a compressing stream that produces gzip-format output,
 using compression level \fIlevel\fR (if specified) which will be an integer
-from 0 to 9.
-'\" TODO: Header dictionary!
+from 0 to 9, and the header descriptor dictionary \fIheader\fR (if specified;
+for keys see \fBzlib gzip\fR).
 .TP
-\fBzlib stream inflate\fR
+\fBzlib stream inflate\fR ?\fB\-dictionary \fIbindata\fR?
 .
 The stream will be a decompressing stream that takes raw compressed input and
 produces uncompressed output.
+.VS "TIP 400"
+If \fIbindata\fR is supplied, it is a compression dictionary to use. Note that
+there are no checks in place to determine whether the compression dictionary
+is correct.
+.VE
 .RE
 .SS "CHECKSUMMING SUBCOMMANDS"
 .TP
@@ -277,10 +319,10 @@ the transformed data.
 The full set of subcommands supported by a streaming instance command,
 \fIstream\fR, is as follows:
 .TP
-\fIstream \fBadd\fR ?\fIoption\fR? \fIdata\fR
+\fIstream \fBadd\fR ?\fIoption...\fR? \fIdata\fR
 .
 A short-cut for
-.QW "\fIstream \fBput \fIoption data\fR"
+.QW "\fIstream \fBput \fR?\fIoption...\fR? \fIdata\fR"
 followed by
 .QW "\fIstream \fBget\fR" .
 .TP
@@ -318,15 +360,27 @@ A short-cut for
 Return up to \fIcount\fR bytes from \fIstream\fR's internal buffers with the
 transformation applied. If \fIcount\fR is omitted, the entire contents of the
 buffers are returned.
+.
+\fIstream \fBheader\fR
+.
+Return the gzip header description dictionary extracted from the stream. Only
+supported for streams created with their \fImode\fR parameter set to
+\fBgunzip\fR.
 .TP
-\fIstream \fBput\fR ?\fIoption\fR? \fIdata\fR
+\fIstream \fBput\fR ?\fIoption...\fR? \fIdata\fR
 .
 Append the contents of the binary string \fIdata\fR to \fIstream\fR's internal
-buffers while applying the transformation. If present, \fIoption\fR must be
-one of the following (or an unambiguous prefix) which are used to modify the
+buffers while applying the transformation. The following \fIoption\fRs are
+supported (or an unambiguous prefix of them), which are used to modify the
 way in which the transformation is applied:
 .RS
 .TP
+\fB\-dictionary\fI binData\fR
+.VS "TIP 400"
+Sets the compression dictionary to use when working with compressing or
+decompressing the data to be \fIbinData\fR.
+.VE
+.TP
 \fB\-finalize\fR
 .
 Mark the stream as finished, ensuring that all bytes have been wholly
@@ -334,12 +388,22 @@ compressed or decompressed. For gzip streams, this also ensures that the
 footer is written to the stream. The stream will need to be reset before
 having more data written to it after this, though data can still be read out
 of the stream with the \fBget\fR subcommand.
+.RS
+.PP
+This option is mutually exclusive with the \fB\-flush\fR and \fB\-fullflush\fR
+options.
+.RE
 .TP
 \fB\-flush\fR
 .
 Ensure that a decompressor consuming the bytes that the current (compressing)
 stream is producing will be able to produce all the bytes that have been
 compressed so far, at some performance penalty.
+.RS
+.PP
+This option is mutually exclusive with the \fB\-finalize\fR and
+\fB\-fullflush\fR options.
+.RE
 .TP
 \fB\-fullflush\fR
 .
@@ -347,6 +411,11 @@ Ensure that not only can a decompressor handle all the bytes produced so far
 (as with \fB\-flush\fR above) but also that it can restart from this point if
 it detects that the stream is partially corrupt. This incurs a substantial
 performance penalty.
+.RS
+.PP
+This option is mutually exclusive with the \fB\-finalize\fR and \fB\-flush\fR
+options.
+.RE
 .RE
 .TP
 \fIstream \fBreset\fR