path: root/tcllib/modules/docstrip/tcldocstrip.dtx

%
% \iffalse
%<*driver>
\documentclass{tclldoc}
\newenvironment{ttdescription}{%
  \description
  \def\makelabel##1{\hspace\labelsep\normalfont\ttfamily ##1}%
}{\enddescription}
\newcommand{\Tcllib}{\textsf{tcllib}}

% The following is a hack, to get around some complications that 
% arise when one tries to use the doc package on two levels 
% simultaneously. Basically, I need to define my own macrocode 
% environment, since at one place I'm going to need it for a block 
% of code that contains the end-of-macrocode string
% "%    \end{macrocode}" (note exactly four spaces).
\makeatletter
\newenvironment{Macrocode}{%
   \macro@code\frenchspacing\@vobeyspaces\xMacro@code
}{\endmacrocode}
% The following piece of code uses the trick of having a macro grab 
% some piece of material and thus ensure that it is tokenized under 
% the expand-time catcodes, even though a different set of catcodes 
% will be in force when that material is actually used. This saves 
% from having to introduce an extra character of \catcode 0, and 
% means only those characters which actually need it (i.e., those 
% inside the final bracket group) are tokenized with unconventional 
% catcodes.
\@firstofone{\bgroup
   \def\@tempa#1{\egroup\def\xMacro@code##1#1{##1\end{Macrocode}}}
   \catcode`\[=1\catcode`\]=2%
   \catcode`\{=12\catcode`\}=12\catcode`\%=12%
   \catcode`\\=13\catcode`\ =13\relax
\@tempa}[%    \end{Macrocode}]
\makeatother
% An easier way around it would be to use a different number of 
% spaces in that particular line, since hardly anyone would notice, 
% but I want the details to be [emph correct].


\begin{document}
\DocInput{tcldocstrip.dtx}
\end{document}
%</driver>
% \fi
% 
% \title{The \textsf{docstrip} \Tcllogo\ package}
% \author{Lars Hellstr\"om}
% \date{25 August 2005}
% \maketitle
% 
% \begin{abstract}
%   The \textsf{docstrip} package provides a pure-\Tcllogo\ 
%   implementation of some of the functionality of the \LaTeX\ 
%   \textsc{docstrip} program. In particular, there is a command 
%   using which one can |source| \Tcllogo\ code from within a 
%   \texttt{.dtx} file.
%   
%   The \textsf{docstrip::util} package provides related 
%   functionality, which is more of interest at installation or 
%   development time than runtime. The main functionality areas are: 
%   (i)~hooks into the \Tcllogo\ package mechanisms, using which one 
%   can avoid depending on the \textsc{docstrip} program for 
%   \Tcllogo\ scripts; (ii)~statistical introspection into source 
%   files; (iii)~alternatives to \LaTeX\ as markup language; 
%   (iv)~patching of source files.
% \end{abstract}
% 
% \changes{1.0}{2004/09/17}{Changing namespace to \texttt{docstrip} and 
%   also all command names. (LH)}
% 
% \tableofcontents
% 
% 
% \section{Usage}
% 
% \subsection{\textsf{docstrip} package}
% 
% The simplest usage of the \textsf{docstrip} package is to source 
% \Tcllogo\ code from within a \texttt{.dtx} file without having to 
% generate any stripped file first. The command that does this is
% \describestring[proc][docstrip]{sourcefrom}|docstrip::sourcefrom|, 
% which has the syntax
% \begin{quote}
%   |docstrip::sourcefrom| \word{filename} \word{terminals} 
%   \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock}
% \end{quote}
% where \word{filename} is the source file name. The \word{terminals} 
% is the list of guard expression terminals that should be considered 
% true; the \textsc{docstrip} program calls these the ``options'' for 
% the source file. The \word{option} and \word{value} arguments are 
% passed on to |fconfigure|, to configure the file before |read|ing 
% it.
% 
% A typical usage is
% \begin{quote}
%   |docstrip::sourcefrom foobar.dtx {foo debug}|
% \end{quote}
% which corresponds to |source|ing the file \texttt{temp.tcl} that 
% would be generated by
% \begin{quote}
%   |\generate{\file{temp.tcl}{\from{foobar.dtx}{foo,debug}}}|
% \end{quote}
% A more advanced usage (making use of the ability to |fconfigure| the 
% source file before reading it) is
% \begin{quote}
%   |docstrip::sourcefrom ruslish.dtx pkg -encoding utf-8|
% \end{quote}
% which ensures that the file is interpreted as being 
% \texttt{utf-8} encoded.
% 
% \iffalse
% (Files which require an encoding specification can actually be tricky 
% to handle using the \textsc{docstrip} program, since most \TeX's will 
% by default write \TeX-style |^^|-escapes for all characters outside 
% visible ASCII, but the \textsf{docstrip} package handles such matters 
% easily.)
% \fi
% 
% The \textsf{docstrip} package can even be used in 
% \texttt{pkgIndex.tcl} scripts. The typical pattern is\pagebreak[2]
%\begin{verbatim}
%   package ifneeded foo 1.0 [format {
%      package require docstrip
%      docstrip::sourcefrom [file join %s foobar.dtx] foo
%   } [list $dir]]
%   package ifneeded bar 0.2 [format {
%      package require docstrip
%      docstrip::sourcefrom [file join %s foobar.dtx] bar
%   } [list $dir]]
%\end{verbatim}
% where |format| is used to embed the package directory into the 
% |package ifneeded| scripts; |list| provides the right amount of 
% quoting of the directory string. Alternatively, one may use 
% |docstrip::util::index_from_catalogue| (see below) to generate such 
% scripts automatically.
% 
% The semantics of |sourcefrom| closely follows those of |source|: The 
% code is evaluated in the local context of the caller, a |return| will 
% abort the sourcing early, and |info script| will return the 
% \word{filename} for the duration of the |sourcefrom|. A difference is 
% that |sourcefrom| does not stop at |\u001a| characters (control-Z, 
% end of file) unless told to by an explicit |-eofchar| option. Also 
% note that the entire file is ``docstripped'' 
% before any of the code in it gets evaluated, so e.g. module nesting 
% errors at the end of the file cannot be hidden by an early |return| 
% in it.
% 
% The actual ``docstripping'' is done by the 
% \describestring[proc][docstrip]{extract}|docstrip::extract| command, 
% which has the syntax
% \begin{quote}
%   |docstrip::extract| \word{text} \word{terminals}
%   \begin{regblock}[\regstar]\word{option} 
%   \word{value}\end{regblock}
% \end{quote}
% Unlike the \textsc{docstrip} program, which is file-oriented, this 
% command takes the \word{text} to extract code from as an argument and 
% returns the code that was extracted. The \word{terminals} is as for 
% |sourcefrom| the list of guard expression terminals that should have 
% the value true. 
% 
% The options are
% \begin{quote}
%   |-annotate| \word{lines}\\
%   |-metaprefix| \word{string}\\
%   |-onerror| \begin{regblock}|throw|\regalt |puts|\regalt
%     |ignore|\end{regblock}\\
%   |-trimlines| \word{boolean}
% \end{quote}
% These control some fine details of the extraction process. See 
% Subsection~\ref{Ssec:Extract} for further information.
% 
% The |extract| command does not as the \textsc{docstrip} program 
% wrap the extracted code up with a preamble and postamble; it just 
% handles the basic extraction, not the higher level operation of 
% complete file generation. The 
% |docstrip::util::modules_from_catalogue| command generates 
% preambles and postambles, however.
% 
% 
% \subsection{\textsf{docstrip::util} package}
% 
% The \textsf{docstrip::util} package is meant for collecting various 
% utility procedures that may be useful for developers who make use of 
% the \textsf{docstrip} package in some projects, either during 
% development or during installation. It is separate from 
% the main package to avoid overhead when |docstrip| is used in 
% |package ifneeded| scripts.
% 
% 
% \subsubsection{Source file introspection}
% 
% \describestring[proc][docstrip::util]{guards}
% The |guards| command collects information about the docstrip guards 
% occurring in a file. It has the subcommands |names|, |counts|, 
% |expressions|, |exprcounts|, and |exprmods| which return 
% information about correct guards in various degrees of detail. The 
% |exprerr| subcommand lists syntactically incorrect guard expressions, 
% and the |rotten| subcommand lists the malformed guard lines.
% 
% \describestring[proc][docstrip::util]{thefile}
% The |thefile| command is a conveniency for reading the contents of 
% a file, since most other \texttt{docstrip::util} commands expect to 
% be handed the text of \texttt{.dtx} or \texttt{.ddt} files as 
% strings (or some other in-memory data structure). It takes as 
% primary argument the name of the file to read, and will like 
% |docstrip::sourcefrom| accept additional option--value pairs for 
% configuring the file channel before reading from it. A final 
% newline is dropped, so that the result can directly be |split| into 
% a list of lines.
% 
% 
% \subsubsection{Package indexing with built-in catalogue}
% 
% \changes{1.3}{2010/04/18}{Renamed the `directory' to `catalogue', 
%   to avoid overloading this term. Should be OK since so far only 
%   one project has contained a directory. (LH)}
% The basic method of installing a \Tcllogo\ package kept in a 
% \texttt{.dtx} file is to run the corresponding \texttt{.ins} file 
% to have \textsc{docstrip} (the program) generate one or several 
% \texttt{.tcl} files, and then use the |pkg_mkIndex| to regenerate 
% the package index file. This introduces a dependency on having 
% \LaTeX\ available when installing however, so one might want to 
% have a pure-\Tcllogo\ alternative. The \textsf{docstrip::util} 
% package provides two: the |index_from_catalogue| and 
% |modules_from_catalogue| commands.
% 
% Parsing \texttt{.ins} files using \Tcllogo\ would be difficult, so 
% the corresponding information about which docstrip modules make up 
% a package is better put somewhere else. It turns out that it can 
% easily be embedded into the \texttt{.dtx} file itself! By 
% convention, the name of this module should be 
% `|docstrip.tcl::catalogue|'---hinting both at who is expected to make 
% use of this information and what it is. The contents of this module 
% make up the \emph{catalogue} of the source file in question.
% 
% \describestring[command]{pkgProvide}
% The main command to use in a catalogue is
% \begin{quote}
%   |pkgProvide| \word{name} \word{version} \word{terminal-list}
% \end{quote}
% which means the module in this file which is selected by the 
% terminals in \word{terminal-list} contains version \word{version} 
% of the package named \word{name}. That module should not contain a 
% |package provide| command, as one will be provided in the 
% |package ifneeded| script.
% 
% \describestring[command]{pkgIndex}
% An older alternative command for identifying package from within 
% a catalogue is
% \begin{quote}
%   |pkgIndex| \word{terminal}\regstar
% \end{quote}
% which means there is a module in the file which should be 
% \emph{indexed} as a package; the package name and version is taken 
% from the |package provide| command(s) found. The module is as usual 
% defined by that the listed \word{terminal}s are true.
% 
% \describestring[command]{fileoptions}
% A configuration command is
% \begin{quote}
%   |fileoptions| \begin{regblock}[\regstar]\word{option}
%   \word{value}\end{regblock}
% \end{quote}
% which sets the |fconfigure| options that will be used when reading 
% the source file. Since the whole file was read into memory just to 
% extract the catalogue, this command works by causing the file to be 
% read again using a new set of options, and it thus has effect for 
% the |pkgProvide| etc.\@ commands following it. (Even if it is 
% perfectly legal, it would be rather strange to use |fileoptions| 
% more than once in a file.)
% 
% A catalogue for this file could be
% \begin{tcl}
%<*docstrip.tcl::catalogue>
pkgIndex pkg
pkgIndex utilpkg
%</docstrip.tcl::catalogue>
% \end{tcl}
% since the two packages include |package provide| commands.
% 
% The |pkgProvide|, |pkgIndex|, and |fileoptions| commands are only 
% available in catalogues. Unknown commands encountered in 
% catalogues are silently ignored.\footnote{
%   This allows for future extensions of the catalogue, with commands 
%   that encode other kinds of entities. One application could be to 
%   list the files of a virtual file system, where the contents of 
%   the individual files are to be extracted from the source file. 
%   Another, perhaps more likely application would be to encode 
%   packages that span several source files.
% }
% 
% \medskip
% 
% \describestring[proc][docstrip::util]{modules_from_catalogue}
% The commands which look at the catalogue of a file are 
% |index_from_catalogue| and |modules_from_catalogue|. The former 
% appends |package ifneeded| commands (which make use of 
% |docstrip::sourcefrom| rather than |source|) to a traditional 
% \texttt{pkgIndex.tcl} file. The latter generates \texttt{.tm} files 
% for the packages (overwriting previous files with the target 
% names). It has the syntax
% \begin{quote}
%   |docstrip::util::modules_from_catalogue| \word{target root} 
%   \word{source file} \begin{regblock}[\regstar] \word{option} 
%   \word{value} \end{regblock}
% \end{quote}
% where the \word{target root} is the directory used as starting 
% point for the paths builts from package names, and 
% \word{source file} is the file to process. The most common
% \word{option}s are:
% \begin{ttdescription}
%   \item[-preamble]
%     Message to put at the top of the generated file. Defaults to 
%     a space (which ends up contributing an empty line).
%   \item[-postamble]
%     Message to put at the bottom of the generated file. Defaults to 
%     being empty.
%   \item[-options]
%     \textsf{Docstrip} expressions terminals in addition to 
%     the basic \texttt{docstrip.tcl::catalogue} to use when 
%     extracting the catalogue. A sort of meta-configuration 
%     facility.
% \end{ttdescription}
% Traditionally, the |-preamble| would be used for a copyright 
% message, but such messages can alternatively be embedded as 
% ``metacomment lines''.
% 
% \describestring[proc][docstrip::util]{index_from_catalogue}
% The syntax of |index_from_catalogue| is
% \begin{quote}
%   |docstrip::util::index_from_catalogue| \word{directory} 
%   \word{pattern} \begin{regblock}[\regstar]\word{option} 
%   \word{value}\end{regblock}
% \end{quote}
% where \word{directory} is the directory whose \texttt{pkgIndex.tcl} 
% file should be amended and \word{pattern} is a |glob|-pattern for 
% files whose \texttt{docstrip.tcl::catalogue}s should be read. The most 
% common \word{option}s are:
% \begin{ttdescription}
%   \item[-recursein]
%     If nonempty, then the operation will be repeated in each 
%     subdirectory matching the pattern specified as \word{value}. 
%     |-recursein *| causes the entire subtree rooted at |-root| to 
%     be processed.
%   \item[-options]
%     \textsf{Docstrip} expression terminals in addition to 
%     the basic \texttt{docstrip.tcl:\nolinebreak[1]:catalogue} to use 
%     when extracting the catalogue; a sort of metaconfiguration 
%     facility.
% \end{ttdescription}
% 
% 
% \subsubsection{Alternative markup languages}
% 
% The |ddt2man| command provides an alternative to \LaTeX\ markup for 
% programmers who think \LaTeX\ is too heavy (e.g.\ installation-wise) 
% and prefer a pure-\Tcllogo\ documentation setup, namely to use 
% \textsf{doctools}~\cite{doctools_fmt} man page markup. This is 
% nowhere near as powerful as \LaTeX, but may well suffice in cases 
% with less sophisticated typographical requirements.
% 
% \describestring[proc][docstrip::util]{ddt2man}
% Since \textsf{doctools} cannot be configured to process 
% docstrip-style master sources directly, a conversion to some format 
% that can be processed is necessary, and that is precisely what the 
% |ddt2man| command does. The syntax is
% \begin{quote}
%   |docstrip::util::ddt2man| \word{ddt-text}
% \end{quote}
% where \word{ddt-text} is the contents of a master source code file 
% and the result is the same text reformatted as \textsf{doctools} 
% man page source. The command name comes from the recommended file 
% suffixes: \textsf{doctools} man pages have the suffix \texttt{.man} 
% and master source files with \textsf{doctools} markup in the comments 
% should use the suffix \texttt{.ddt} to distinguish them from 
% \texttt{.dtx} files which have \LaTeX\ markup in the comments.
% 
% A typical usage might be
%\begin{verbatim}
%package require docstrip::util
%package require doctools
%doctools::new man2html -format html
%set ddt [docstrip::util::thefile somefile.ddt]
%set man [docstrip::util::ddt2man $ddt]
%set html [man2html format $man]
%\end{verbatim}
% after which the |html| variable contains ordinary HTML code.
%
% 
% \subsubsection{Patching sources}
% 
% \describestring[proc][docstrip::util]{patch}
% The |patch| command is a still slightly experimental utility for 
% applying patches against extracted files to the master sources 
% proper; it works by translating extracted file line numbers to 
% master source file numbers and applies differences at the translated 
% positions. Currently the text being patched is kept in memory as a 
% list of lines, but this may change if this feature is more closely 
% integrated with the \Tcllogo lib diff file utilies offered by the 
% \textsf{rcs} package. |patch| also has a companion command 
% \describestring[proc][docstrip::util]{import_unidiff}
% |import_unidiff| that translates patches to the format understood by 
% the |patch| command.
% 
% An example of using these commands would be
%\begin{verbatim}
%set sourceL [split [docstrip::util::thefile somefile.dtx] \n]
%set generated [docstrip::util::thefile foobar.tcl]
%set diff [docstrip::util::thefile foobar.patch]
%set conflicts [docstrip::util::patch sourceL {foo bar} $generated\
%  [docstrip::util::import_unidiff $diff]]
%\end{verbatim}
% after which one in principle can overwrite \texttt{somefile.dtx} 
% with the result of |join $sourceL \n|, but more often one should 
% rather send these patched contents to a text editor for further 
% review. For one thing, there may be conflicts. For another, it is 
% often necessary to also update the comment lines around modified 
% sections.
% 
% 
% \section{Headers}
% 
% The guiding principle for the various file headers has been to 
% collect all occurrencies of a version number in the same place. 
% This is not entirely possible, since the |manpage_begin| and 
% |require| manpage commands both contain the package version 
% numbers, but at least it is possible to collect |require|s, 
% |package require|s, and |package provide|s together.
% \changes{1.3}{2010/04/30}{File headers interleaved, to simplify 
%   changing version numbers. (LH)}
% 
% Since this leaves |manpage_begin| of the \textsf{doctools} manpages 
% the odd man out, we'd better begin with that.
% \begin{tcl}
%<*man,utilman>
%<man>[manpage_begin docstrip n 1.2]
%<utilman>[manpage_begin docstrip_util n 1.3]
%<man>[see_also docstrip_util]
%<utilman>[see_also docstrip]
%<utilman>[see_also doctools]
%<utilman>[see_also doctools_fmt]
%<utilman>[keywords .ddt]
[keywords .dtx]
%<utilman>[keywords catalogue]
%<utilman>[keywords diff]
[keywords docstrip]
%<utilman>[keywords doctools]
[keywords documentation]
[keywords LaTeX]
[keywords {literate programming}]
%<utilman>[keywords module]
%<utilman>[keywords {package indexing}]
%<utilman>[keywords patch]
[keywords source]
%<utilman>[keywords {Tcl module}]
[copyright "2003\u20132010 Lars Hellstr\u00F6m\
  <Lars dot Hellstrom at residenset dot net>"]
[moddesc {Literate programming tool}]
%<man>[titledesc {Docstrip style source code extraction}]
%<utilman>[titledesc {Docstrip-related utilities}]
[category  {Documentation tools}]
%</man,utilman>
% \end{tcl}
% The other files involved in this great interleaving are the actual 
% packages (\Module{pkg} and \Module{utilpkg}) and a hardcoded index 
% file (\Module{idx}).
% \changes{1.1}{2005/02/26}{Added \texttt{pkgIndex.tcl} source. 
%   (LH, after suggestion by AK)}
% 
% \Tcllogo~8.4 is required because |info script| with an argument 
% is used by the |sourcefrom| command, and there are also some uses 
% of the |eq| operator in |if| expressions.
% \begin{tcl}
%<pkg,utilpkg>package require Tcl 8.4
%<man,utilman>[require Tcl 8.4]
%<idx>if {![package vsatisfies [package provide Tcl] 8.4]} {return}
% \end{tcl}
% That version number check is the only part of the 
% \texttt{pkgIndex.tcl} file that would not be done the same way by 
% the standard |pkg_mkIndex| command.
% 
% Next comes the \textsf{docstrip} package version.
% \begin{tcl}
%<pkg>package provide docstrip 1.2
%<idx>package ifneeded docstrip 1.2\
%<idx>  [list source [file join $dir docstrip.tcl]]
%<man,utilman>[require docstrip [opt 1.2]]
%<utilpkg>package require docstrip 1.2
% \end{tcl}
% The \textsf{docstrip::util} package has a dependency on the 
% \textsf{docstrip} package, but not the other way around. Hence the 
% next block is slightly shorter.
% \begin{tcl}
%<utilpkg>package provide docstrip::util 1.3.1
%<idx>package ifneeded docstrip::util 1.3.1\
%<idx>  [list source [file join $dir docstrip_util.tcl]]
%<utilman>[require docstrip::util [opt 1.3]]
% \end{tcl}
% This ends the interleaved parts of the headers.
% 
% The following is a trick to use non-ASCII characters in manpages 
% without having to put them as such in the source: when an emdash 
% (U+2014) is needed, just write |[vset emdash]|.
% \begin{tcl}
%<*man,utilman>
%<-ASCII>[vset emdash \u2014]
%<+ASCII>[vset emdash --]
[description]
%</man,utilman>
% \end{tcl}
% The \Module{ASCII} guard here makes it possible to fall back to 
% simpler encodings, on platforms which require it, but the default 
% is the proper emdash.
% 
% The public commands in both packages are exported. This is 
% meaningful mostly for the \textsf{docstrip::util} package, which 
% imports |extract| from \textsf{docstrip}. The corresponding 
% |namespace import| will however occur further down, to ensure that 
% a combined file extracted with |pkg,utilpkg| works too.
% \changes{1.2}{2005/06/20}{Added namespace export code. (LH)}
% \begin{tcl}
%<*pkg>
namespace eval docstrip {
   namespace export extract sourcefrom
}
%</pkg>
%<*utilpkg>
namespace eval docstrip::util {
   namespace export ddt2man guard patch thefile\
     packages_provided index_from_catalogue modules_from_catalogue\
     classical_preamble classical_postamble
}
%</utilpkg>
% \end{tcl}
% 
% \subsection{Test headers}
% 
% What now remains to initialise are only the tests, but that is a 
% slightly complicated affair, since it means interacting with the 
% \Tcllib\ test harness. Originally the code didn't do that, 
% so there is a \Module{tcllibtest} terminal which can be used to 
% select whether to try it or not.
% 
% The original route was to use the \textsf{tcltest} package provided 
% by the \Tcllogo\ package mechanism, but explicitly |source| the 
% \texttt{docstrip.tcl} file in the same directory as the 
% \texttt{docstrip.test} being run. That meant you could have one 
% \textsf{docstrip} version installed and run tests on another.
% \changes{1.2}{2005/09/18}{Introduced the 
%    \texttt{docstrip\_sources\_dir} variable as the directory in 
%    which to search for \texttt{docstrip.tcl}, 
%    \texttt{docstrip\_util.tcl}, and \texttt{tcldocstrip.dtx}. 
%    Using \texttt{file normalize} to compute it. (LH)}
% \begin{tcl}
%<*test,utiltest>
%<*!tcllibtest>
package require tcltest 2
variable docstrip_sources_dir\
  [file dirname [file normalize [info script]]]
source [file join $docstrip_sources_dir docstrip.tcl]
puts "** Has Tcl docstrip package (v [package provide docstrip]) **"
%<*utiltest>
source [file join $docstrip_sources_dir docstrip_util.tcl]
puts "** Has Tcl docstrip::util package\
  (v [package provide docstrip::util]) **"
%</utiltest>
%</!tcllibtest>
% \end{tcl}
% 
% The \Tcllib\ set-up instead begins with |source|ing 
% \texttt{..\slash devtools\slash testutilities.tcl}, which 
% (in that directory structure) will be a file that causes 
% \textsf{tcltest} to be loaded. The other commands below are also 
% provided by that file.
% \changes{1.2.1}{2006/09/13}{Modified the setup of the testsuite 
%   to match the other modules and packages in \Tcllib. (AK)}
% \begin{tcl}
%<*tcllibtest>
source [file join\
  [file dirname [file dirname [file join [pwd] [info script]]]]\
  devtools testutilities.tcl]
testsNeedTcl     8.4
testsNeedTcltest 2
testing {useLocal docstrip.tcl docstrip}
%<utiltest>testing {useLocal docstrip_util.tcl docstrip::util}
variable docstrip_sources_dir [localPath {}]
%</tcllibtest>
% \end{tcl}
% One of the tests require that \texttt{tcldocstrip.dtx} (this file) and 
% \texttt{docstrip.tcl} are both present. A \textsf{tcltest} constraint 
% is declared for this purpose.
% \begin{tcl}
tcltest::testConstraint docstripSourcesAvailable [expr {[
   file exists [file join $docstrip_sources_dir docstrip.tcl]
] && [
   file exists [file join $docstrip_sources_dir tcldocstrip.dtx]
]}]
%</test,utiltest>
% \end{tcl}
% 
% 
% \part{The docstrip package}
% 
% \setnamespace{docstrip}
% 
% Here follows the source both for the actual package and its manpage, 
% the latter of which is in four sections: introduction, 
% description of the format of files to be processed by 
% \textsc{docstrip}, description of commands, and basic remarks on 
% overall document structure. Since command descriptions and 
% implementations appear in the same sections of the \texttt{.dtx} 
% file, a big batch of manpage source has to appear first.
% 
% 
% \section{Manpage}
% 
% The introduction is indended for \Tcllogo\ programmers who have not
% previously encountered \textsc{docstrip}---hence it is probably a 
% bit boring for experienced \LaTeX\ programmers.
%    \begin{macrocode}
%<*man>

[syscmd Docstrip] is a tool created to support a brand of Literate 
Programming. It is most common in the (La)TeX community, where it 
is being used for pretty much everything from the LaTeX core and up, 
but there is nothing about [syscmd docstrip] which prevents using it 
for other types of software.
[para]

In short, the basic principle of literate programming is that program 
source should primarily be written and structured to suit the 
developers (and advanced users who want to peek "under the hood"), not 
to suit the whims of a compiler or corresponding source code consumer. 
This means literate sources often need some kind of "translation" to an 
illiterate form that dumb software can understand. 
The [package docstrip] Tcl package handles this translation.
[para]

Even for those who do not whole-hartedly subscribe to the philosophy 
behind literate programming, [syscmd docstrip] can bring greater 
clarity to in particular:
[list_begin itemized]
  [item] programs employing non-obvious mathematics
  [item] projects where separate pieces of code, perhaps in 
    different languages, need to be closely coordinated.
[list_end]
The first is by providing access to much more powerful typographical 
features for source code comments than are possible in plain text. 
The second is because all the separate pieces of code can be kept 
next to each other in the same source file.
[para]

The way it works is that the programmer edits directly only one or 
several "master" source code files, from which [syscmd docstrip] 
generates the more traditional "source" files compilers or the like 
would expect. The master sources typically contain a large amount of 
documentation of the code, sometimes even in places where the code 
consumers would not allow any comments. The etymology of "docstrip" 
is that this [emph doc]umentation was [emph strip]ped away (although 
"code extraction" might be a better description, as it has always 
been a matter of copying selected pieces of the master source rather 
than deleting text from it). 
The [package docstrip] Tcl package contains a reimplementation of 
the basic extraction functionality from the [syscmd docstrip] 
program, and thus makes it possible for a Tcl interpreter to read 
and interpret the master source files directly.
[para]

Readers who are not previously familiar with [syscmd docstrip] but 
want to know more about it may consult the following sources.
[list_begin enumerated]
[enum]
  [emph {The tclldoc package and class}],
  [uri {http://ctan.org/tex-archive/macros/latex/contrib/tclldoc/}].
[enum]
  [emph {The DocStrip utility}],
  [uri {http://ctan.org/tex-archive/macros/latex/base/docstrip.dtx}].
[enum]
  [emph {The doc and shortvrb Packages}],
  [uri {http://ctan.org/tex-archive/macros/latex/base/doc.dtx}].
[enum]
  Chapter 14 of
  [emph {The LaTeX Companion}] (second edition),
  Addison-Wesley, 2004; ISBN 0-201-36299-6.
[list_end]
%    \end{macrocode}
% 
% \subsection{File format}
% 
% In order to keep some kind of document structure in this file, it is 
% best that the manpage sections are present also in the \LaTeX\ table 
% of contents.
% 
%    \begin{macrocode}

[section {File format}]

The basic unit [syscmd docstrip] operates on are the [emph lines] of 
a master source file. Extraction consists of selecting some of these 
lines to be copied from input text to output text. The basic 
distinction is that between [emph {code lines}] (which are copied and 
do not begin with a percent character) and [emph {comment lines}] 
(which begin with a percent character and are not copied).

[example {
%</man>
%    \end{macrocode}
% 
% At this point, let's do a little trick: use this example also as the 
% first test. This is just a matter of putting groups of lines in the 
% right modules.
% \begin{tcl}
%<*test>
tcltest::test docstrip-1.1 {code/comment line distinction} -body {
%</test>
%<*test,man>
   docstrip::extract [join {
     {% comment}
     {% more comment !"#$%&/(}
     {some command}
     { % blah $blah "Not a comment."}
     {% abc; this is comment}
     {# def; this is code}
     {ghi}
     {% jkl}
   } \n] {}
%<man>}]
%<man>returns the same sequence of lines as
%<man>[example {
%<test>} -result [
   join {
     {some command}
     { % blah $blah "Not a comment."}
     {# def; this is code}
     {ghi} ""
   } \n
%<test>]
%</test,man>
% \end{tcl}
% This completes the code for the test, so let's switch back to just 
% \Module{man}.
%    \begin{macrocode}
%<*man>
}]

It does not matter to [syscmd docstrip] what format is used for the 
documentation in the comment lines, but in order to do better than 
plain text comments, one typically uses some markup language. Most 
commonly LaTeX is used, as that is a very established standard and 
also provides the best support for mathematical formulae, but the 
[package docstrip::util] package also gives some support for 
[term doctools]-like markup.
[para]

Besides the basic code and comment lines, there are also 
[emph {guard lines}], which begin with the two characters '%<', and 
[emph {meta-comment lines}], which begin with the two characters 
'%%'. Within guard lines there is furthermore the distinction between 
[emph {verbatim guard lines}], which begin with '%<<', and ordinary 
guard lines, where the '%<' is not followed by another '<'. The last 
category is by far the most common.
[para]

Ordinary guard lines conditions extraction of the code line(s) they 
guard by the value of a boolean expression; the guarded block of 
code lines will only be included if the expression evaluates to true. 
The syntax of an ordinary guard line is one of
[example {
    '%' '<' STARSLASH EXPRESSION '>'
    '%' '<' PLUSMINUS EXPRESSION '>' CODE
}]
where
[example {
    STARSLASH  ::=  '*' | '/'
    PLUSMINUS  ::=  | '+' | '-'
    EXPRESSION ::= SECONDARY | SECONDARY ',' EXPRESSION
                 | SECONDARY '|' EXPRESSION
    SECONDARY  ::= PRIMARY | PRIMARY '&' SECONDARY
    PRIMARY    ::= TERMINAL | '!' PRIMARY | '(' EXPRESSION ')'
    CODE       ::= { any character except end-of-line }
}]
Comma and vertical bar both denote 'or'. Ampersand denotes 'and'. 
Exclamation mark denotes 'not'. A TERMINAL can be any nonempty string 
of characters not containing '>', '&', '|', comma, '(', or ')', 
although the [syscmd docstrip] manual is a bit restrictive and only 
guarantees proper operation for strings of letters (although even 
the LaTeX core sources make heavy use also of digits in TERMINALs). 
The second argument of [cmd docstrip::extract] is the list of those 
TERMINALs that should count as having the value 'true'; all other 
TERMINALs count as being 'false' when guard expressions are evaluated.
[para]

In the case of a '%<*[emph EXPRESSION]>' guard, the lines guarded are 
all lines up to the next '%</[emph EXPRESSION]>' guard with the same 
[emph EXPRESSION] (compared as strings). The blocks of code delimited 
by such '*' and '/' guard lines must be properly nested.
%    \end{macrocode}
% This looks like a good place for another example.
% \begin{tcl}
[example {
%</man>
%<*man,test>
%<test>tcltest::test docstrip-1.2 {blocks and nesting} -body {
   set text [join {
      {begin}
      {%<*foo>}
      {1}
      {%<*bar>}
      {2}
      {%</bar>}
      {%<*!bar>}
      {3}
      {%</!bar>}
      {4}
      {%</foo>}
      {5}
      {%<*bar>}
      {6}
      {%</bar>}
      {end}
   } \n]
   set res [docstrip::extract $text foo]
   append res [docstrip::extract $text {foo bar}]
   append res [docstrip::extract $text bar]
%<*man>
}]
sets $res to the result of
[example {
%</man>
%<test>} -result [
   join {
      {begin}
      {1}
      {3}
      {4}
      {5}
      {end}
      {begin}
      {1}
      {2}
      {4}
      {5}
      {6}
      {end}
      {begin}
      {5}
      {6}
      {end} ""
   } \n
%<test>]
%</man,test>
%<*man>
}]
% \end{tcl}
%    \begin{macrocode}

In guard lines without a '*', '/', '+', or '-' modifier after the 
'%<', the guard applies only to the CODE following the '>' on that 
single line. A '+' modifier is equivalent to no modifier. A '-' 
modifier is like the case with no modifier, but the expression is 
implicitly negated, i.e., the CODE of a '%<-' guard line is only 
included if the expression evaluates to false.
[para]

Metacomment lines are "comment lines which should not be stripped 
away", but be extracted like code lines; these are sometimes used for 
copyright notices and similar material. The '%%' prefix is however 
not kept, but substituted by the current [option -metaprefix], which 
is customarily set to some "comment until end of line" character (or 
character sequence) of the language of the code being extracted.
%    \end{macrocode}
% Ho hum, another example\slash test.
%    \begin{macrocode}
[example {
%</man>
%<*man,test>
%<*test>
tcltest::test docstrip-1.3 {plusminus guards and metacomments} -body {
%</test>
   set text [join {
      {begin}
      {%<foo> foo}
      {%<+foo>plusfoo}
      {%<-foo>minusfoo}
      {middle}
      {%% some metacomment}
      {%<*foo>}
      {%%another metacomment}
      {%</foo>}
      {end}
   } \n]
   set res [docstrip::extract $text foo -metaprefix {# }]
   append res [docstrip::extract $text bar -metaprefix {#}]
%<*man>
}]
sets $res to the result of
[example {
%</man>
%<test>} -result [
   join {
      {begin}
      { foo}
      {plusfoo}
      {middle}
      {#  some metacomment}
      {# another metacomment}
      {end}
      {begin}
      {minusfoo}
      {middle}
      {# some metacomment}
      {end} ""
   } \n
%<test>]
%</man,test>
%<*man>
}]

Verbatim guards can be used to force code line 
interpretation of a block of lines even if some of them happen to look 
like any other type of lines to docstrip. A verbatim guard has the 
form '%<<[emph END-TAG]' and the verbatim block is terminated by the 
first line that is exactly '%[emph END-TAG]'. 
[example {
%</man>
%<*man,test>
%<*test>
tcltest::test docstrip-1.4 {verbatim mode} -body {
%</test>
   set text [join {
      {begin}
      {%<*myblock>}
      {some stupid()}
      {   #computer<program>}
      {%<<QQQ-98765}
      {% These three lines are copied verbatim (including percents}
      {%% even if -metaprefix is something different than %%).}
      {%</myblock>}
      {%QQQ-98765}
      {   using*strange@programming<language>}
      {%</myblock>}
      {end}
   } \n]
   set res [docstrip::extract $text myblock -metaprefix {# }]
   append res [docstrip::extract $text {}]
%<*man>
}]
sets $res to the result of
[example {
%</man>
%<test>} -result [
   join {
      {begin}
      {some stupid()}
      {   #computer<program>}
      {% These three lines are copied verbatim (including percents}
      {%% even if -metaprefix is something different than %%).}
      {%</myblock>}
      {   using*strange@programming<language>}
      {end}
      {begin}
      {end} ""
   } \n
%<test>]
%</man,test>
%<*man>
}]
The processing of verbatim guards takes place also inside blocks of 
lines which due to some outer block guard will not be copied.
[para]

The final piece of [syscmd docstrip] syntax is that extraction 
stops at a line that is exactly "\endinput"; this is often used to 
avoid copying random whitespace at the end of a file. In the unlikely 
case that one wants such a code line, one can protect it with a 
verbatim guard.
%    \end{macrocode}
% Thus far the general descriptions; now for the actual commands. 
% The manpage source for these are next to the actual implementations.
%    \begin{macrocode}

[section Commands]

The package defines two commands.

[list_begin definitions]
%    \end{macrocode}
% 
% \section{Command implementations}
% 
% \subsection{Code extraction}
% \label{Ssec:Extract}
% 
% \begin{proc}{extract}
%   The |extract| procedure implements the core functionality of the 
%   \textsc{docstrip} program: copying the some lines of code as 
%   directed by relevant guard linnes. The main difference is that this 
%   takes the input as a string and returns output as a string. Each 
%   line in the return value ends with a newline.
%   
%   The syntax is
%   \begin{quote}
%     |docstrip::extract| \word{text} \word{terminal list}
%     \begin{regblock}[\regstar]\word{option} 
%     \word{value}\end{regblock}
%   \end{quote}
%   where \word{text} is the string to docstrip and \word{terminal list} 
%   is the list of expression terminals that should be true. 
%   \changes{1.0}{2004/09/30}{Switched to option--value syntax for 
%     equivalents of \textsc{docstrip} parameters. (LH)}
%   The options are
%   \begin{quote}
%     |-annotate| \word{lines}\\
%     |-metaprefix| \word{string}\\
%     |-onerror| \begin{regblock}|throw|\regalt |puts|\regalt
%       |ignore|\end{regblock}\\
%     |-trimlines| \word{boolean}
%   \end{quote}
%   \changes{1.2}{2005/06/16}{Added \texttt{-annotate} option. (LH)}
%   
%   The \describeopt[docstrip]{extract}{-metaprefix}|-metaprefix| value 
%   is the string to use for the \textsc{docstrip} parameter 
%   \verb|\MetaPrefix|. The default is `|%%|'. 
%   The \describeopt[docstrip]{extract}{-trimlines}|-trimlines| option 
%   specifies whether spaces at the end of a line should be trimmed 
%   away before it is processed. For compatibility with 
%   \textsc{docstrip} (which due to a quirk in the low-level input 
%   routines of \TeX\ cannot help doing this), this is by default on.
%   
%   The \describeopt[docstrip]{extract}{-annotate}|-annotate| option 
%   modifies the output format, so that each extracted line is followed 
%   by \word{lines} lines of annotation information. These extra lines 
%   have the following format
%   \begin{quote}
%     \word{type} \word{offprefix} \word{onprefix}\\
%     \meta{lineno}\\
%     \meta{current stack}
%   \end{quote}
%   If \word{lines} is |0| then none of the above lines is included. If 
%   \word{lines} is |1| then only the first line is included. If 
%   \word{lines} is |2| then the first two lines are included. Finally 
%   if \word{lines} is |3| then all three lines are included. The 
%   behaviour for other values of \word{lines} is unspecified. The 
%   default value is |0|.
%   
%   A first annotation line is a list of three elements. The first 
%   element is a ``line type'', the second element is the prefix string 
%   that was removed from the line (an empty string if nothing was 
%   removed), and the third element is the prefix that was added to the 
%   line (either the |-metaprefix| value or an empty string). The line 
%   type is one of: |V|~(verbatim), |M|~(metacomment), |+|~(plus or no 
%   modifier guard line), |-|~(minus modifier guard line), and 
%   |.|~(normal line). The second annotation line is simply the current 
%   input line number. The third annotation line is the current block 
%   guard stack---a list of guard expression strings.
%   
%   The \describeopt[docstrip]{extract}{-onerror}|-onerror| option 
%   specifies what should happen when an error in the \word{text} being 
%   processed is detected. The value |puts| causes error messages to 
%   be written to |stderr|, but processing continues. |ignore| causes 
%   processing to continue silently. The default |throw| causes a 
%   \Tcllogo\ error to be thrown. In this last case, the |errorCode| is 
%   set to a list with the format
%   \begin{quote}
%     |DOCSTRIP| \word{situation} \word{lineno}
%   \end{quote}
%   where \word{lineno} is the line number (starting at one) of the line 
%   where the error was detected. The \word{situation}s are described 
%   below, at the positions in the code where they are detected.
%   
%   Now, for the manpage, a quick resum\'e of the above.
%    \begin{macrocode}
[call [cmd docstrip::extract] [arg text] [arg terminals] [
   opt "[arg option] [arg value] ..."
]]
  The [cmd extract] command docstrips the [arg text] and returns the 
  extracted lines of code, as a string with each line terminated with 
  a newline. The [arg terminals] is the list of those guard 
  expression terminals which should evaluate to true. 
  The available options are:
  [list_begin options]
  [opt_def -annotate [arg lines]]
    Requests the specified number of lines of annotation to follow 
    each extracted line in the result. Defaults to 0. Annotation lines 
    are mostly useful when the extracted lines are to undergo some 
    further transformation. A first annotation line is a list of three 
    elements: line type, prefix removed in extraction, and prefix 
    inserted in extraction. The line type is one of: 'V' (verbatim), 
    'M' (metacomment), '+' (+ or no modifier guard line), '-' (- 
    modifier guard line), '.' (normal line). A second annotation line 
    is the source line number. A third annotation line is the current 
    stack of block guards. Requesting more than three lines of 
    annotation is currently not supported.
  [opt_def -metaprefix [arg string]]
    The string by which the '%%' prefix of a metacomment line will 
    be replaced. Defaults to '%%'. For Tcl code this would typically 
    be '#'.
  [opt_def -onerror [arg keyword]]
    Controls what will be done when a format error in the [arg text] 
    being processed is detected. The settings are:
    [list_begin definitions]
    [def [const ignore]]
      Just ignore the error; continue as if nothing happened.
    [def [const puts]]
      Write an error message to [const stderr], then continue 
      processing.
    [def [const throw]]
      Throw an error. The [option -errorcode] is set to a list whose 
      first element is [const DOCSTRIP], second element is the 
      type of error, and third element is the line number where 
      the error is detected. This is the default.
    [list_end]
  [opt_def -trimlines [arg boolean]]
    Controls whether [emph spaces] at the end of a line should be 
    trimmed away before the line is processed. Defaults to true.
  [list_end]
  
  It should be remarked that the [arg terminals] are often called 
  "options" in the context of the [syscmd docstrip] program, since 
  these specify which optional code fragments should be included.

%</man>
%    \end{macrocode}
%   Hmm\dots\ Perhaps not so quick, after all.
%   \begin{tcl}
%<*pkg>
proc docstrip::extract {text terminals args} {
   array set O {
      -annotate 0
      -metaprefix %%
      -onerror throw
      -trimlines 1
   }
   array set O $args
%   \end{tcl}
%   The |O| array is for options of this procedure. The |T| array is 
%   for the terminals, so that the truth value of a terminal can be 
%   tested using |info exists|.
%   \begin{tcl}
   foreach t $terminals {set T($t) ""}
%   \end{tcl}
%   |stripped| is where the text that passes docstripping is collected.
%   \begin{tcl}
   set stripped ""
%   \end{tcl}
%   |block_stack| is the list of modules inside which the current line 
%   lies. |offlevel| is the number of modules that must be exited 
%   before code lines should once again be included. |verbatim| is a 
%   flag for whether verbatim mode is in force.
%   \begin{tcl}
   set block_stack [list]
   set offlevel 0
   set verbatim 0
%   \end{tcl}
%   |lineno| is the input line number counter, for use in error 
%   messages. The first line in the file has number |1|.
%   \begin{tcl}
   set lineno 0
%   \end{tcl}
%   Here starts the main loop over lines in the \word{text}. It 
%   constitutes the majority of the procedure and is split in two 
%   parts. The smaller part handles lines in verbatim mode (unusual), 
%   the large part handles lines in normal mode (with comment lines, 
%   code lines, guard lines, and so on). |continue| is being used in 
%   this loop to skip generation of annotation lines, for those branches 
%   that do not contribute a line to the output in the first place.
%   \begin{tcl}
   foreach line [split $text \n] {
      incr lineno
      if {$O(-trimlines)} then {
         set line [string trimright $line " "]
      }
      if {$verbatim} then {
         if {$line eq $endverbline} then {
            set verbatim 0
            continue
         } elseif {$offlevel} then {
            continue
         }
         append stripped $line \n
         if {$O(-annotate)>=1} then {append stripped {V "" ""} \n}
      } else {
%   \end{tcl}
%   Here starts the processing of lines in non-verbatim mode.
%   \begin{tcl}
         switch -glob -- $line %%* {
            if {!$offlevel} then {
               append stripped $O(-metaprefix)\
                 [string range $line 2 end] \n
               if {$O(-annotate)>=1} then {
                  append stripped [list M %% $O(-metaprefix)] \n
               }
            }
         } %<<* {
            set endverbline "%[string range $line 3 end]"
            set verbatim 1
            continue
         } %<* {
%   \end{tcl}
%   This is the case of an ordinary guard line, which accounts for most 
%   of the complexities in the file format. Here one can also encounter 
%   a number of conditions which constitute errors in the data being 
%   processed. The first of these is the 
%   \describestring[error situation]{BADGUARD}|BADGUARD| 
%   \word{situation}: the line looks like a guard line, but there is no 
%   |>| terminating the guard expression.
%   \begin{tcl}
            if {![
               regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\
                 modifier expression line
            ]} then {
               extract,error BADGUARD\
                 "Malformed guard \"\n$line\n\""
                 "Malformed guard on line $lineno"
               continue
            }
%   \end{tcl}
%   At this point, an ordinary guard line has successfully been split 
%   into parts. First the expression is evaluated, by converting it 
%   to an |expr| expression.
%   \begin{tcl}
            regsub -all -- {\\|\{|\}|\$|\[|\]| |;} $expression\
              {\\&} E
            regsub -all -- {,} $E {|} E
            regsub -all -- {[^()|&!]+} $E {[info exists T(&)]} E
            if {[catch {expr $E} val]} then {
               extract,error EXPRERR\
                 "Error in expression <$expression> ignored"\
                 "docstrip: $val"
               set val -1
            }
%   \end{tcl}
%   If |$E| isn't a valid |expr| expression, then the original guard 
%   expression must have been malformed. That is an 
%   \describestring[error situation]{EXPRERR}|EXPRERR| \word{situation}.
%   \changes{1.0}{2004/09/29}{Catching errors in expressions. (LH)}
%   
%   With the expression evaluated, the processing of a guard line
%   now branches according to its type.
%   \begin{tcl}
            switch -exact -- $modifier * {
               lappend block_stack $expression
               if {$offlevel || !$val} then {incr offlevel}
               continue
            } / {
               if {![llength $block_stack]} then {
%   \end{tcl}
%   In this case there was no open block for this guard to end. That 
%   is a \describestring[error situation]{SPURIOUS}|SPURIOUS| 
%   \word{situation}.
%   \begin{tcl}
                  extract,error SPURIOUS\
                    "Spurious end block </$expression> ignored"\
                    "Spurious end block </$expression>"
               } else {
                  if {[string compare $expression\
                    [lindex $block_stack end]]} then {
%   \end{tcl}
%   In this case the expression of the block being closed does not match 
%   the expression on the block on top of the stack. That is a 
%   \describestring[error situation]{MISMATCH}|MISMATCH| 
%   \word{situation}. \textsc{docstrip} by default raises an error and 
%   recovers by treating this situation as a typo.
%   \begin{tcl}
                     extract,error MISMATCH\
                       "Found </$expression> instead of\
                       </[lindex $block_stack end]>"
                  }
%   \end{tcl}
%   All that error processing makes it easy to lose track, but the 
%   following two lines are what does the real work for an end of block 
%   guard: pop a block off the stack and decrement the |offlevel|.
%   \begin{tcl}
                  if {$offlevel} then {incr offlevel -1}
                  set block_stack [lreplace $block_stack end end]
               }
               continue
%   \end{tcl}
%   These last cases of the |switch| handle |-|, |+|, and ``no 
%   modifier'' lines.
%   \begin{tcl}
            } - {
               if {$offlevel || $val} then {continue}
               append stripped $line \n
               if {$O(-annotate)>=1} then {
                  append stripped [list - %<-${expression}> ""] \n
               }
            } default {
               if {$offlevel || !$val} then {continue}
               append stripped $line \n
               if {$O(-annotate)>=1} then {
                  append stripped\
                    [list + %<${modifier}${expression}> ""] \n
               }
            }
         } %* {continue}\
%   \end{tcl}
%   Back to the outer |switch|. With comment lines, nothing is done. 
%   A line being the exact string |\endinput| terminates the stripping.
%   \begin{tcl}
         {\\endinput} {
           break
         } default {
%   \end{tcl}
%   Other lines are code lines. These are included or not, depending on 
%   the |offlevel|.
%   \begin{tcl}
            if {$offlevel} then {continue}
            append stripped $line \n
            if {$O(-annotate)>=1} then {append stripped {. "" ""} \n}
         }
      }
%   \end{tcl}
%   Finally there is the code for annotation lines two and above.
%   \begin{tcl}
      if {$O(-annotate)>=2} then {append stripped $lineno \n}
      if {$O(-annotate)>=3} then {append stripped $block_stack \n}
   }
   return $stripped
}
%   \end{tcl}
% 
%   \begin{proc}{extract,error}
%     Since the |extract| procedure can detect many different 
%     errors which should all go through roughtly the same handling, 
%     the common parts of that have been factored out into this 
%     |extract,error| procedure. It accesses the variable |lineno| and 
%     array element |O(-onerror)| in the local context of its caller 
%     to determine the current line number and error reporting mode. 
%     Apart from that, the call syntax is
%     \begin{quote}
%       |docstrip::extract,error| \word{situation} \word{message} 
%       \word{error message}\regopt
%     \end{quote}
%     where \word{situation} is what would be used to identify the
%     error in |errorCode| (if |-onerror| is |throw|), \word{message} 
%     is the message that would be written to |stderr| (if |-onerror| 
%     is |puts|), and \word{error message} is the error message to use 
%     (if |-onerror| is |throw|). The default for \word{error message} 
%     is the \word{message}. Neither \word{message} nor \word{error 
%     message} should end with a period, as such punctuation may be 
%     provided by |extract,error|.
%     \changes{1.1}{2005/02/27}{Procedure factored out from 
%       \texttt{extract}, as suggested by AK. (LH)}
%     \begin{tcl}
proc docstrip::extract,error {situation message {errmessage ""}} {
   upvar 1 O(-onerror) onerror lineno lineno
   switch -- [string tolower $onerror] "puts" {
      puts stderr "docstrip: $message on line $lineno."
   } "ignore" {} default {
      if {$errmessage ne ""} then {
         error $errmessage "" [list DOCSTRIP $situation $lineno]
      } else {
         error $message "" [list DOCSTRIP $situation $lineno]
      }
   }
}
%</pkg>
%     \end{tcl}
%   \end{proc}
% \end{proc}
% 
% The following tests annotation. It is mostly the same code as in the 
% verbatim mode test.
% \begin{tcl}
%<*test>
tcltest::test docstrip-1.5 {annotation} -body {
   set text [join {
      {begin}
      {%<*myblock>}
      {some stupid()}
      {%<foo>   #computer<program>}
      {%<<QQQ-98765}
      {% These three lines are copied verbatim (including percents}
      {%% even if -metaprefix is something different than %%).}
      {%</myblock>}
      {%QQQ-98765}
      {   using*strange@programming<language>}
      {%</myblock>}
      {%%end}
   } \n]
   docstrip::extract $text {myblock foo} -metaprefix {# } -annotate 3
} -result [
   join {
      {begin} {. "" ""} 1 {}
      {some stupid()} {. "" ""} 3 myblock
      {   #computer<program>} {+ %<foo> {}} 4 myblock
      {% These three lines are copied verbatim (including percents}
        {V "" ""} 6 myblock
      {%% even if -metaprefix is something different than %%).}
        {V "" ""} 7 myblock
      {%</myblock>} {V "" ""} 8 myblock
      {   using*strange@programming<language>} {. "" ""} 10 myblock
      {# end} {M %% {# }} 12 {}
      ""
   } \n
]
% \end{tcl}
% 
% The following is a test of the |extract| procedure, which compares its 
% output to the \textsc{docstrip} program output. If need be, and \LaTeX\ 
% is not available, then this could also be modified to produce a new 
% version of \texttt{docstrip.tcl} using the |extract| command of 
% an older version.
% \begin{tcl}
tcltest::test docstrip-2.1 {have docstrip extract itself} -constraints {
   docstripSourcesAvailable
} -body {
   # First read in the ready-stripped file, but gobble the preamble and
   # postamble, as those are a bit messy to reproduce.
   set F [open [file join $docstrip_sources_dir docstrip.tcl] r]
   regsub -all -- {(^|\n)#[^\n]*} [read $F] {} stripped
   close $F
   # Then read the master source and strip it manually.
   set F [open [file join $docstrip_sources_dir tcldocstrip.dtx] r]
   set source [read $F]
   close $F
   set stripped2 [docstrip::extract $source pkg -metaprefix ##]
   # Finally compare the two.
   if {[string trim $stripped \n] ne [string trim $stripped2 \n]} then {
      error "$strippped\n ne \n$stripped2"
   }
}
%</test>
% \end{tcl}
%
% 
% 
% \subsection{Code sourcing}
% 
% \begin{proc}{sourcefrom}
%   This procedure behaves as a docstripping |source| command: it reads 
%   a file, docstrips its contents in memory, and evaluates the result 
%   as a \Tcllogo\ script in the context of the caller. The syntax is
%   \begin{quote}
%     |docstrip::sourcefrom| \word{filename} \word{terminals} 
%     \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock}
%   \end{quote}
%   where \word{filename} is the file name and \word{terminals} is the 
%   list of true guard expression terminals. The \word{option} and 
%   \word{value} arguments are passed on to |fconfigure|, to configure 
%   the file before |read|ing it.
%   \changes{1.0}{2004/10/01}{Added \texttt{info script} management. 
%     (LH)}
%   \begin{tcl}
%<*man>
[call [cmd docstrip::sourcefrom] [arg filename] [arg terminals] [
   opt "[arg option] [arg value] ..."
]]
  The [cmd sourcefrom] command is a docstripping emulation of 
  [cmd source]. It opens the file [arg filename], reads it, closes it, 
  docstrips the contents as specified by the [arg terminals], and 
  evaluates the result in the local context of the caller, during 
  which time the [cmd info] [method script] value will be the 
  [arg filename]. The options are passed on to [cmd fconfigure] to 
  configure the file before its contents are read. The 
  [option -metaprefix] is set to '#', all other [cmd extract] 
  options have their default values.
%</man>
%<*pkg>
proc docstrip::sourcefrom {name terminals args} {
   set F [open $name r]
   if {[llength $args]} then {
      eval [linsert $args 0 fconfigure $F]
   }
   set text [read $F]
   close $F
   set oldscr [info script]
   info script $name
   set code [catch {
      uplevel 1 [extract $text $terminals -metaprefix #]
   } res]
   info script $oldscr
   if {$code == 1} then {
      error $res $::errorInfo $::errorCode
   } else {
      return $res
   }
}
%</pkg>
%   \end{tcl}
% \end{proc}
% 
% Testing the above procedure requires an external file. The business 
% with |info script| is to check that this is getting set and reset 
% correctly. The business with the |baz| variable tests that the file 
% contents are being evaluated in the context calling |sourcefrom|.
% \changes{1.2}{2005/10/02}{Moddified test to make it work when 
%    tmpdir is not the current directory. (LH)}
% \begin{tcl}
%<*test>
tcltest::test docstrip-2.2 {soucefrom} -setup {
   set dtxname [tcltest::makeFile [join {
      {% Just a minor test file.}
      {puts A}
      {%<*bar>}
      {puts B}
      {%<*foo>}
      {puts [info exists baz]}
      {set baz 1}
      {%</foo>}
      {%<-foo>return}
      {%</bar>}
      {puts $baz}
      {puts [file tail [info script]]}
      {%<*!foo>}
      {puts C}
      "%% Tricky comment; guess what comes next\\"
      {%</!foo>}
      {incr baz}
% \end{tcl}
% What the above construction does depends on the truth value of |foo|. 
% When true, the \Module{!foo} block is skipped in its entirety, and 
% thus the next command after |puts [file tail [info script]]| is 
% |incr baz|. However when |foo| is false the block will be included. 
% The metacomment line gets a prefix |#| and will therefore become 
% a comment when the code is evaluated. The backslash escapes the 
% subsequent newline, and thus the |incr baz| will only be part of 
% a \Tcllogo\ comment.
% \begin{tcl}
      {puts "baz=$baz"}
   } \n] te27st01.dtx]
} -body {
   set baz 0
   puts [info script]
   docstrip::sourcefrom $dtxname {foo bar}
   puts [info script]
   docstrip::sourcefrom $dtxname {}
   docstrip::sourcefrom $dtxname {bar}
   puts $baz
} -cleanup {
   tcltest::removeFile $dtxname
} -output [join [list\
   [info script]\
   {A} {B} {1} {1} {te27st01.dtx} {baz=2}\
   [info script]\
   {A} {2} {te27st01.dtx} {C} {baz=2}\
   {A} {B}\
   {2} ""
] \n]
%</test>
% \end{tcl}
% 
% 
% 
% \section{Manpage section on document structure}
% 
% This completes the package code, but there are more things which
% should be said on the manpage.
% 
% 
%    \begin{macrocode}
%<*man>
[list_end]


[section {Document structure}]

The file format (as described above) determines whether a master
source code file can be processed correctly by [syscmd docstrip], 
but the usefulness of the format is to no little part also dependent
on that the code and comment lines together constitute a well-formed
document.
[para]

For a document format that does not require any non-Tcl software, see 
the [cmd ddt2man] command in the [package docstrip::util] package. It 
is suggested that files employing that document format are given the 
suffix [file .ddt], to distinguish them from the more traditional 
LaTeX-based [file .dtx] files.
[para]

Master source files with [file .dtx] extension are usually set up so
that they can be typeset directly by [syscmd latex] without any
support from other files. This is achieved by beginning the file 
with the lines
[example_begin]
   % \iffalse
   %<*driver>
   \documentclass{tclldoc}
   \begin{document}
   \DocInput{[emph filename.dtx]}
   \end{document}
   %</driver>
   % \fi
[example_end]
or some variation thereof. The trick is that the file gets read twice.
With normal LaTeX reading rules, the first two lines are comments and 
therefore ignored. The third line is the document preamble, the fourth
line begins the document body, and the sixth line ends the document, 
so LaTeX stops there [vset emdash] non-comments below that point in 
the file are never subjected to the normal LaTeX reading rules. Before 
that, however, the \DocInput command on the fifth line is processed, 
and that does two things: it changes the interpretation of '%' from 
"comment" to "ignored", and it inputs the file specified in the 
argument (which is normally the name of the file the command is in). 
It is this second time that the file is being read that the comments 
and code in it are typeset.
[para]

The function of the \iffalse ... \fi is to skip lines two to seven 
on this second time through; this is similar to the "if 0 { ... }" 
idiom for block comments in Tcl code, and it is needed here because 
(amongst other things) the \documentclass command may only be 
executed once. The function of the <driver> guards is to prevent this 
short piece of LaTeX code from being extracted by [syscmd docstrip]. 
The total effect is that the file can function both as a LaTeX 
document and as a [syscmd docstrip] master source code file.
[para]

It is not necessary to use the tclldoc document class, but that does 
provide a number of features that are convenient for [file .dtx] 
files containing Tcl code. More information on this matter can be 
found in the references above.

%</man>
%    \end{macrocode}
% 
% 
% \part{The docstrip utilities package}
% 
% The |extract| command is used by several \textsf{docstrip::util} 
% commands, so it is imported.
% \begin{tcl}
%<*utilpkg>
namespace eval docstrip::util {
   namespace import [namespace parent]::extract
}
%</utilpkg>
% \end{tcl}
% \setnamespace{docstrip::util}
% 
%    \begin{macrocode}
%<*utilman>
The [package docstrip::util] package is meant for collecting various 
utility procedures that are mainly useful at installation or 
development time. It is separate from the base package to avoid 
overhead when the latter is used to [cmd source] code.
[para]
%</utilman>
%    \end{macrocode}
% 
% \section{Package indexing and generation}
% 
% Manually writing \texttt{pkgIndex.tcl} files for 
% \textsf{docstrip}-encoded packages gets boring after a while 
% (especially since they will have to be updated after every version 
% increment), so one would like to automate this task. The following 
% implements a mechanism for this that parallels the standard 
% |pkg_mkIndex| command.
% 
% 
% \subsection{The catalogue}
% 
% The main difference between a \textsf{docstrip} file and an 
% ordinary \texttt{.tcl} file is that it is not clear to the casual 
% reader what modules in a file should be combined to make a directly 
% sourceable file. This information can however be encoded as a 
% separate module into the source file itself.
% 
% The special module holding catalogue information will be
% \begin{quote}
%   \Module{docstrip.tcl::catalogue}
% \end{quote}
% The \texttt{docstrip.tcl} prefix here is intended as a clear 
% indication of who is meant to read this information. The contents 
% of this module will be \Tcllogo\ code that causes some embedded 
% file to be stripped, sourced, and indexed.
% The catalogue code will be evaluated in a separate safe interpreter, 
% so that a somewhat controlled set of commands can be made available.
% 
% \begin{variable}{thefile}
% \begin{variable}{filename}
%   The variables |thefile| and |filename| are used by 
%   \textsf{docstrip} catalogue commands as sources of information 
%   about the current file. |thefile| is the actual file contents, 
%   whereas |filename| is the name of the file (including a path, if 
%   one is needed).
% \end{variable}\end{variable}
% 
% \begin{variable}{fileoptions}
%   This variable holds the list of |fconfigure|-options for 
%   configuring a file before reading it. This information must be 
%   remembered, because it needs to be recorded in generated 
%   |package ifneeded| scripts.
% \end{variable}
% 
% \begin{proc}{fileoptions}
%   This command may be used in \textsf{docstrip} directories to 
%   change the set of options for files. The call syntax is
%   \begin{quote}
%     |fileoptions| \begin{regblock}[\regstar]\word{option}
%     \word{value}\end{regblock}
%   \end{quote}
%   and the current set of options is set to precisely those 
%   specified (old options are forgotten). There is no particular 
%   return value, but the |thefile| variable contents are updated to 
%   reflect the new |fileoptions|.
%   
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::fileoptions {args} {
   variable filename
   variable thefile [eval [list thefile $filename] $args]
   variable fileoptions $args
}
%</utilpkg>
%   \end{tcl}
% \end{proc}
% 
%    \begin{macrocode}
%<*utilman>
[section {Package indexing commands}]

Like raw [file .tcl] files, code lines in docstrip source files can 
be searched for package declarations and corresponding indices 
constructed. A complication is however that one cannot tell from the 
code blocks themselves which will fit together to make a working 
package; normally that information would be found in an accompanying 
[file .ins] file, but parsing one of those is not an easy task. 
Therefore [package docstrip::util] introduces an alternative encoding 
of such information, in the form of a declarative Tcl script: the 
[term catalogue] (of the contents in a source file).
[para]

The special commands which are available inside a catalogue are:
[list_begin definitions]
[call [cmd pkgProvide] [arg name] [arg version] [arg terminals]]
  Declares that the code for a package with name [arg name] and 
  version [arg version] is made up from those modules in the source 
  file which are selected by the [arg terminals] list of guard 
  expression terminals. This code should preferably not contain a 
  [cmd {package}] [method {provide}] command for the package, as one 
  will be provided by the package loading mechanisms.
[call [cmd pkgIndex] [opt "[arg terminal] ..."]]
  Declares that the code for a package is made up from those modules 
  in the source file which are selected by the listed guard 
  expression [arg terminal]s. The name and version of this package is 
  determined from [cmd {package}] [method {provide}] command(s) found 
  in that code (hence there must be such a command in there).
[call [cmd fileoptions] [opt "[arg option] [arg value] ..."]]
  Declares the [cmd fconfigure] options that should be in force when 
  reading the source; this can usually be ignored for pure ASCII 
  files, but if the file needs to be interpreted according to some 
  other [option -encoding] then this is how to specify it. The 
  command should normally appear first in the catalogue, as it takes 
  effect only for commands following it.
[list_end]
Other Tcl commands are supported too [vset emdash] a catalogue is 
parsed by being evaluated in a safe interpreter [vset emdash] but they 
are rarely needed. To allow for future extensions, unknown commands 
in the catalogue are silently ignored.
[para]

To simplify distribution of catalogues together with their source 
files, the catalogue is stored [emph {in the source file itself}] as 
a module selected by the terminal '[const docstrip.tcl::catalogue]'. 
This supports both the style of collecting all catalogue lines in one 
place and the style of putting each catalogue line in close proximity 
of the code that it declares.
[para]

%    \end{macrocode}
%    \DontCheckModules
%    \begin{Macrocode}
Putting catalogue entries next to the code they declare may look as 
follows
[example {
%<<verbatim
%    First there's the catalogue entry
%    \begin{tcl}
%<docstrip.tcl::catalogue>pkgProvide foo::bar 1.0 {foobar load}
%    \end{tcl}
%    second a metacomment used to include a copyright message
%    \begin{macrocode}
%<*foobar>
%% This file is placed in the public domain.
%    \end{macrocode}
%    third the package implementation
%    \begin{tcl}
namespace eval foo::bar {
   # ... some clever piece of Tcl code elided ...
%    \end{tcl}
%    which at some point may have variant code to make use of a 
%    |load|able extension
%    \begin{tcl}
%<*load>
   load [file rootname [info script]][info sharedlibextension]
%</load>
%<*!load>
   # ... even more clever scripted counterpart of the extension
   # also elided ...
%</!load>
}
%</foobar>
%    \end{tcl}
%    and that's it!
%verbatim
}]
The corresponding set-up with [cmd pkgIndex] would be
[example {
%<<verbatim
%    First there's the catalogue entry
%    \begin{tcl}
%<docstrip.tcl::catalogue>pkgIndex foobar load
%    \end{tcl}
%    second a metacomment used to include a copyright message
%    \begin{tcl}
%<*foobar>
%% This file is placed in the public domain.
%    \end{tcl}
%    third the package implementation
%    \begin{tcl}
package provide foo::bar 1.0
namespace eval foo::bar {
   # ... some clever piece of Tcl code elided ...
%    \end{tcl}
%    which at some point may have variant code to make use of a 
%    |load|able extension
%    \begin{tcl}
%<*load>
   load [file rootname [info script]][info sharedlibextension]
%</load>
%<*!load>
   # ... even more clever scripted counterpart of the extension
   # also elided ...
%</!load>
}
%</foobar>
%    \end{tcl}
%    and that's it!
%verbatim
%    \end{Macrocode}
%    \CheckModules
%    \begin{macrocode}
}]
%</utilman>
%    \end{macrocode}
% 
% \begin{variable}{Report}
%   Since commands in the catalogue will often be implemented as 
%   doing something, there is a need for giving them a way of 
%   reporting back what they did, as the basic ``report in return 
%   value'' idiom doesn't work for scripts. Hence there is a variable 
%   |Report| where information can be gathered. The value of this 
%   list is a list to which new items can be appended, although in 
%   the end they will typically be |join|ed with a newline as 
%   separator (thus blurring the distinction between a multiline item 
%   and multiple one-line items).
% \end{variable}
% 
% \begin{proc}{Report}
%   Normally, items are contributed to the report using the call
%   \begin{quote}
%     |Report| \word{item}
%   \end{quote}
%   where the \word{item} is some human-readable string.
%   \begin{variable}{Report_store}
%   \begin{variable}{Report_cmd}
%     A complication is that one sometimes wants reports to be 
%     returned by the top level command, but other times one wants 
%     them to be written to the controlling terminal immediately 
%     (e.g. to give feedback of progress). The |Report| mechanism 
%     aims to support both by having the action of the |Report| 
%     command controlled by one boolean variables |Report_store| and 
%     one command prefix |Report_cmd|. If the former is true, then 
%     the \word{item} is appended to the |Report| list. Moreover 
%     the latter is evaluated with the \word{item} as an extra 
%     argument. To have the latter ``do nothing'', use |list|.
%   \end{variable}\end{variable}
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::Report {item} {
   variable Report_store
   if {$Report_store} then {
      variable Report
      lappend Report $item
   }
   variable Report_cmd
   eval [linsert $Report_cmd end $item]
}
%</utilpkg>
%   \end{tcl}
% \end{proc}
% 
% 
% \subsection{Index entry generation}
% 
% \begin{proc}{index_from_catalogue}
%   The |index_from_catalogue| command generates package index data 
%   by reading catalogue modules in master source files and appends 
%   these entries to the relevant \texttt{pkgIndex.tcl} file. The 
%   call syntax is
%   \changes{1.3}{2010/04/20}{Promoted the directory to being 
%     a mandatory argument, for symmetry with 
%     \texttt{pkg\PrintChar{95}mkIndex}. (LH)}
%   \begin{quote}
%     |docstrip::util::index_from_catalogue| \word{directory} 
%     \word{pattern} \begin{regblock}[\regstar]\word{option} 
%     \word{value}\end{regblock}
%   \end{quote}
%   where \word{directory} is the directory whose 
%   \texttt{pkgIndex.tcl} should be updated and \word{pattern} is a 
%   |glob|-pattern for files to read catalogues in. Currently the 
%   following \word{option}s are implemented:
%   \begin{ttdescription}
%     \item[-recursein]
%       If nonempty, then the operation will be repeated in each 
%       subdirectory matching the pattern specified as \word{value}. 
%       |-recursein *| causes the entire subtree rooted at the 
%       \word{directory} to be processed.
%     \item[-options]
%       \textsf{Docstrip} expressions terminals in addition to 
%       the basic \texttt{docstrip.tcl::catalogue} to use when 
%       extracting the catalogue; a sort of meta-configuration 
%       facility.
%     \item[-sourceconf]
%       |fconfigure| options applied to the source file, before 
%       reading. |fileoptions| commands in the catalogue will 
%       override this setting (completely replacing the set of 
%       options); this will primarily control what is used when 
%       extracting the catalogue. Defaults to empty.
%     \item[-report]
%       Takes a boolean value. If true, the report will be the return 
%       value of |index_from_catalogue|. Defaults to false, in which 
%       case there is no particular return value.
%     \item[-reportcmd]
%       Takes a command prefix as value, which will be called as
%       \begin{quote}
%         \meta{prefix} \word{item}
%       \end{quote}
%       for every \word{item} being reported. Defaults to 
%       |puts stdout|; use |list| to effectively disable this feature. 
%       The return value from the prefix is ignored.
%     \item[-RecursionDepth]
%       An internal option used when making a recursive call to 
%       signal the distance to the top invokation. A positive value 
%       means ``don't bother about initialising the |Report| system.''
%   \end{ttdescription}
%   
%    \begin{macrocode}
%<*utilman>
[list_begin definitions]
[call [cmd docstrip::util::index_from_catalogue] [arg dir]\
  [arg pattern] [opt "[arg option] [arg value] ..."]]
  This command is a sibling of the standard [cmd pkg_mkIndex] 
  command, in that it adds package entries to [file pkgIndex.tcl] 
  files. The difference is that it indexes [syscmd docstrip]-style 
  source files rather than raw [file .tcl] or loadable library files. 
  Only packages listed in the catalogue of a file are considered.
  [para]
  
  The [arg dir] argument is the directory in which to look for files 
  (and whose [file pkgIndex.tcl] file should be amended).
  The [arg pattern] argument is a [cmd glob] pattern of files to look 
  into; a typical value would be [const *.dtx] or 
  [const *.{dtx,ddt}]. Remaining arguments are option-value pairs, 
  where the supported options are:
  [list_begin options]
  [opt_def -recursein [arg dirpattern]]
    If this option is given, then the [cmd index_from_catalogue] 
    operation will be repeated in each subdirectory whose name 
    matches the [arg dirpattern]. [option -recursein] [const *] will 
    cause the entire subtree rooted at [arg dir] to be indexed.
  [opt_def -sourceconf [arg dictionary]]
    Specify [cmd fileoptions] to use when reading the catalogues of 
    files (and also for reading the packages if the catalogue does 
    not contain a [cmd fileoptions] command). Defaults to being 
    empty. Primarily useful if your system encoding is very different 
    from that of the source file (e.g., one is a two-byte encoding 
    and the other is a one-byte encoding). [const ascii] and 
    [const utf-8] are not very different in that sense.
  [opt_def -options [arg terminals]]
    The [arg terminals] is a list of terminals in addition to 
    [const docstrip.tcl::catalogue] that should be held as true when 
    extracting the catalogue. Defaults to being empty. This makes it 
    possible to make use of "variant sections" in the catalogue 
    itself, e.g. gaurd some entries with an extra "experimental" and 
    thus prevent them from appearing in the index unless that is 
    generated with "experimental" among the [option -options].
  [opt_def -report [arg boolean]]
    If the [arg boolean] is true then the return value will be a 
    textual, probably multiline, report on what was done. Defaults 
    to false, in which case there is no particular return value.
  [opt_def -reportcmd [arg commandPrefix]]
    Every item in the report is handed as an extra argument to the 
    command prefix. Since [cmd index_from_catalogue] would typically 
    be used at a rather high level in installation scripts and the 
    like, the [arg commandPrefix] defaults to 
    "[cmd puts] [const stdout]". 
    Use [cmd list] to effectively disable this feature. The return 
    values from the prefix are ignored.
  [list_end]
  
  The [cmd {package ifneeded}] scripts that are generated contain 
  one [cmd {package require docstrip}] command and one 
  [cmd docstrip::sourcefrom] command. If the catalogue entry was 
  of the [cmd pkgProvide] kind then the [cmd {package ifneeded}] 
  script also contains the [cmd {package provide}] command.
  [para]
  
  Note that [cmd index_from_catalogue] never removes anything from an 
  existing [file pkgIndex.tcl] file. Hence you may need to delete it 
  (or have [cmd pkg_mkIndex] recreate it from scratch) before running 
  [cmd index_from_catalogue] to update some piece of information, such 
  as a package version number.
  [para]
%</utilman>
%    \end{macrocode}
%   
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::index_from_catalogue {dir pattern args} {
   array set O {
      -options "" 
      -sourceconf "" 
      -report 0
      -reportcmd {puts stdout}
      -RecursionDepth 0
   }
   array set O $args
   if {$O(-RecursionDepth)==0} then {
      variable Report {}  Report_store $O(-report) \
        Report_cmd $O(-reportcmd)
   }
%   \end{tcl}
%   The first step is to make sure that there is a 
%   \texttt{pkgIndex.tcl} file to append to.
%   \begin{tcl}
   set targetFn [file join $dir pkgIndex.tcl]
   Report "Entries will go to: $targetFn"
   if {![file exists $targetFn]} then {
      Report "Generating empty index file."
      set F [open $targetFn w]
      puts $F {# Tcl package index file, version 1.1}
      puts $F {# This file is generated by the "pkg_mkIndex" command}
      puts $F {# and sourced either when an application starts up or}
      puts $F {# by a "package unknown" script.  It invokes the}
      puts $F {# "package ifneeded" command to set up package-related}
      puts $F {# information so that packages will be loaded automatically}
      puts $F {# in response to "package require" commands.  When this}
      puts $F {# script is sourced, the variable $dir must contain the}
      puts $F {# full path name of this file's directory.}
      close $F
   }
%   \end{tcl}
%   The second step is to gather the |package ifneeded| scripts for 
%   the directory in question. This involves creating a temporary helper 
%   interpreter for parsing the \Module{docstrip.tcl::catalogue}.
%   \begin{tcl}
   set c [interp create -safe]
   $c eval {
      proc unknown args {}
   }
   $c alias pkgProvide [namespace which PkgProvide]
   $c alias pkgIndex [namespace which PkgIndex]
   $c alias fileoptions [namespace which fileoptions]
   variable PkgIndex ""
   foreach fn [glob -nocomplain -directory $dir -tails $pattern] {
      Report "Processing file: $fn"
      variable filename [file join $dir $fn]
      variable fileoptions $O(-sourceconf)
      variable thefile [eval [list thefile $filename] $fileoptions]
      set catalogue [extract $thefile\
        [linsert $O(-options) 0 docstrip.tcl::catalogue]\
        -metaprefix {#} -onerror puts]
      $c eval $catalogue
   }
   interp delete $c
%   \end{tcl}
%   The third step is easy: append the gathered material to the file. 
%   A header is inserted that records the |-options| and 
%   |-sourceconf| settings that were used.
%   \begin{tcl}
   if {$PkgIndex ne ""} then {
      set F [open $targetFn {WRONLY APPEND}]
      set cmd [list docstrip::util::index_from_catalogue $dir $pattern]
      if {$O(-options) ne ""} then {
         lappend cmd -options $O(-options)
      }
      if {$O(-sourceconf) ne ""} then {
         lappend cmd -sourceconf $O(-sourceconf)
      }
      puts $F "\n## Appendix generated by:\n##  $cmd$PkgIndex"
      close $F
   }
%   \end{tcl}
%   Finally, the procedure may recurse into subdirectories and do the 
%   same things there.
%   \begin{tcl}
   if {[info exists O(-recursein)]} then {
      incr O(-RecursionDepth)
      foreach fn [
         glob -nocomplain -tails -types d -directory $dir\
           $O(-recursein)
      ] {
         eval [list index_from_catalogue [file join $dir $fn] $pattern]\
           [array get O]
      }
   }
   if {$O(-RecursionDepth)==0 && $O(-report)} then {
      return [join $Report \n]
   }
}
%   \end{tcl}
% \end{proc}
% 
% \begin{variable}{PkgIndex}
%   The |PkgIndex| variable stores material that should be written 
%   to the \texttt{pkgIndex.tcl} file. The |PkgIndex| procedure 
%   appends suitable |package ifneeded| commands to it. Each command 
%   must have a newline in front of it.
% \end{variable}
% 
% \begin{proc}{PkgProvide}
%   The |PkgProvide| procedure is an implementation of the |pkgProvide| 
%   command in \textsf{docstrip} directories. It generates 
%   |package ifneeded| commands and appends them to the |PkgIndex| 
%   variable. The call syntax is
%   \begin{quote}
%     |pkgProvide| \word{pkg-name} \word{version} \word{terminal-list}
%   \end{quote}
%   where the \word{terminal}s are the true terminals in guard 
%   expressions. There is no particular return value.
%   
%   The |package ifneeded| scripts generated have the form
%   \begin{quote}
%     |package provide |\word{pkg-name} \word{version}\\
%     |package require docstrip|\\
%     |docstrip::sourcefrom |\word{filename} \word{terminal-list}
%     \meta{fileoptions}
%   \end{quote}
%   (except that semicolons rather than newlines are used as command 
%   separators). That the |package provide| command gets embedded in 
%   the script like may seem unintuitive, but the same thing is done 
%   in the |package ifneeded| scripts generated for \texttt{.tm} 
%   files. Also note that the \word{filename} must be constructed 
%   when the index file is |source|d; this mix of static and dynamic 
%   data leads to a certain amount of Quoting Hell.
%   
%   First, better check that the \word{version} is valid.
%   \begin{tcl}
proc docstrip::util::PkgProvide {pkg ver terminals} {
   if {[catch {package vcompare 0 $ver}]} then {
      Report "Malformed version number $ver given for package $pkg."
      return
   }
   variable PkgIndex
   variable filename
   variable fileoptions
%   \end{tcl}
%   Since command substitution will have to happen inside the script 
%   argument of |package ifneeded|, that word is quote-delimited. The 
%   previous words are straightforwardly handled by |list|-quoting.
%   \begin{tcl}
   append PkgIndex \n [list package ifneeded $pkg $ver] { "}
%   \end{tcl}
%   The |package provide| command is fixed and can thus be handled by 
%   a |list|-quoting here and now, but since that |list|-quoted 
%   string is then embedded into a quote-delimited word, any 
%   characters in it that trigger substitutions or terminate the word 
%   must be escaped. However, there will only be such characters 
%   around for very bizarre choices of package name.
%   \begin{tcl}
   append PkgIndex [string map {\\ {\\} \$ {\$} \[ {\[} \" {\"}}\
     [list package provide $pkg $ver]] {; }
%   \end{tcl}
%   The |package require docstrip| command is at least harmless.
%   \begin{tcl}
   append PkgIndex {package require docstrip} {; }
%   \end{tcl}
%   But for the |docstrip::sourcefrom| command, a different technique 
%   is used. Here, there will be a quoting |list| command present in 
%   the \texttt{pkgIndex.tcl} file, to be evaluated when that is 
%   |source|d, and therefore the quote-delimited nature of the 
%   enclosing word becomes irrelevant; a simple |list|-quoting of 
%   data to be embedded as command arguments is again sufficient.
%   \begin{tcl}
   append PkgIndex {[list docstrip::sourcefrom }\
     {[file join $dir } [list [file tail $filename]] {] }\
     [linsert $fileoptions 0 $terminals] {]"}
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{PkgIndex}
%   The |PkgIndex| procedure is an implementation of the |pkgIndex| 
%   command in \textsf{docstrip} directories. It generates 
%   |package ifneeded| commands and appends them to the |PkgIndex| 
%   variable. The call syntax is
%   \begin{quote}
%     |pkgIndex| \word{terminal}\regstar
%   \end{quote}
%   where the \word{terminal}s are the true terminals in guard 
%   expressions. There is no particular return value.
%   
%   \begin{tcl}
proc docstrip::util::PkgIndex {args} {
   variable thefile
   if {[catch {
      packages_provided [extract $thefile $args -metaprefix {#}]
   } res]} then {
      if {[lindex $::errorCode 0] eq "DOCSTRIP"} then {
         Report "Stripping error \"$res\"\nwhile indexing module\
           <[join $args ,]>."
      } else {
         Report "Code evaluation error:\n  $res\nwhile indexing\
           module <[join $args ,]>."
      }
   } else {
      variable filename
      variable PkgIndex
      variable fileoptions
      foreach {pkg ver} $res {
         append PkgIndex \n [list package ifneeded $pkg $ver] { "}
         append PkgIndex {package require docstrip} {; }
         append PkgIndex {[list docstrip::sourcefrom }\
           {[file join $dir } [list [file tail $filename]] {] }\
           [linsert $fileoptions 0 $args] {]"}
      }
   }
}
%</utilpkg>
%   \end{tcl}
% \end{proc}
% 
% 
% 
% \subsection{Module generation}
% 
% An alternative to package indices is to create \Tcllogo\ 
% Module~(\texttt{.tm}) files.
% 
% \begin{proc}{modules_from_catalogue}
%   This procedure scans the \Module{docstrip.tcl::catalogue} of a 
%   \texttt{.dtx} file and writes out \Tcllogo\ module files for the 
%   packages it finds. The call syntax is
%   \begin{quote}
%     |modules_from_catalogue| \word{target root} \word{source file}
%     \begin{regblock}[\regstar] \word{option} \word{value} 
%     \end{regblock}
%   \end{quote}
%   where the \word{target root} is the directory used as starting 
%   point for the paths builts from package names are generated, and 
%   \word{source file} is the file to process. The supported 
%   \word{option}s are:
%   \begin{ttdescription}
%     \item[-formatpostamble]
%       Command prefix used to format postamble messages. The call 
%       syntax is
%       \begin{quote}
%         \meta{prefix} \word{message} \word{target filename} 
%         \word{source filename} \word{terminal-list} 
%       \end{quote}
%       and the return value is the formatted message. Defaults to 
%       |classical_postamble {##}|.
%     \item[-formatpreamble]
%       Command prefix used to format preamble messages. The call 
%       syntax is
%       \begin{quote}
%         \meta{prefix} \word{message} \word{target filename} 
%         \word{source filename} \word{terminal-list} 
%       \end{quote}
%       and the return value is the formatted message. Defaults to 
%       |classical_preamble {##}|.
%     \item[-options]
%       \textsf{Docstrip} expressions terminals in addition to 
%       the basic \texttt{docstrip.tcl::catalogue} to use when 
%       extracting the catalogue. A sort of meta-configuration 
%       facility.
%     \item[-postamble]
%       Message to put at the top of the generated file. Defaults to 
%       being empty. See also |-formatpostamble|.
%     \item[-preamble]
%       Message to put at the top of the generated file. Defaults to 
%       a space (which ends up contributing an empty line). See also 
%       |-formatpreamble|.
%     \item[-report]
%       Takes a boolean value. If true, the report will be the return 
%       value of |modules_from_catalogue|. If false, there is no 
%       particular return value. The default is true.
%     \item[-reportcmd]
%       Takes a command prefix as value, which will be called as
%       \begin{quote}
%         \meta{prefix} \word{item}
%       \end{quote}
%       for every \word{item} being reported. Defaults to |list|, 
%       which effectively disables this feature. 
%       The return value from the prefix is ignored.
%     \item[-sourceconf]
%       |fconfigure| options applied to the source file, before 
%       reading.
%   \end{ttdescription}
%   
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::modules_from_catalogue] [arg target]\
  [arg source] [opt "[arg option] [arg value] ..."]]
  This command is an alternative to [cmd index_from_catalogue] which 
  creates Tcl Module ([file .tm]) files rather than 
  [file pkgIndex.tcl] entries. Since this action is more similar to 
  what [syscmd docstrip] classically does, it has features for 
  putting pre- and postambles on the generated files.
  [para]
  
  The [arg source] argument is the name of the source file to 
  generate [file .tm] files from. The [arg target] argument is the 
  directory which should count as a module path, i.e., this is what 
  the relative paths derived from package names are joined to. The 
  supported options are:
  [list_begin options]
  [opt_def -preamble [arg message]]
    A message to put in the preamble (initial block of comments) of 
    generated files. Defaults to a space. May be several lines, which 
    are then separated by newlines. Traditionally used for copyright 
    notices or the like, but metacomment lines provide an alternative 
    to that. 
  [opt_def -postamble [arg message]]
    Like [option -preamble], but the message is put at the end of the 
    file instead of the beginning. Defaults to being empty.
  [opt_def -sourceconf [arg dictionary]]
    Specify [cmd fileoptions] to use when reading the catalogue of 
    the [arg source] (and also for reading the packages if the 
    catalogue does not contain a [cmd fileoptions] command). Defaults 
    to being empty. Primarily useful if your system encoding is very 
    different from that of the source file (e.g., one is a two-byte 
    encoding and the other is a one-byte encoding). [const ascii] and 
    [const utf-8] are not very different in that sense.
  [opt_def -options [arg terminals]]
    The [arg terminals] is a list of terminals in addition to 
    [const docstrip.tcl::catalogue] that should be held as true when 
    extracting the catalogue. Defaults to being empty. This makes it 
    possible to make use of "variant sections" in the catalogue 
    itself, e.g. gaurd some entries with an extra "experimental" guard 
    and thus prevent them from contributing packages unless those are 
    generated with "experimental" among the [option -options].
  [opt_def -formatpreamble [arg commandPrefix]]
    Command prefix used to actually format the preamble. Takes four 
    additional arguments [arg message], [arg targetFilename], 
    [arg sourceFilename], and [arg terminalList] and returns a fully 
    formatted preamble. Defaults to using [cmd classical_preamble] 
    with a [arg metaprefix] of '##'.
  [opt_def -formatpostamble [arg commandPrefix]]
    Command prefix used to actually format the postamble. Takes four 
    additional arguments [arg message], [arg targetFilename], 
    [arg sourceFilename], and [arg terminalList] and returns a fully 
    formatted postamble. Defaults to using [cmd classical_postamble] 
    with a [arg metaprefix] of '##'.
  [opt_def -report [arg boolean]]
    If the [arg boolean] is true (which is the default) then the return 
    value will be a textual, probably multiline, report on what was 
    done. If it is false then there is no particular return value.
  [opt_def -reportcmd [arg commandPrefix]]
    Every item in the report is handed as an extra argument to this 
    command prefix. Defaults to [cmd list], which effectively disables 
    this feature. The return values from the prefix are ignored. Use 
    for example "[cmd puts] [const stdout]" to get report items 
    written immediately to the terminal.
  [list_end]
  An existing file of the same name as one to be created will be 
  overwritten.
%</utilman>
%    \end{macrocode}
%   
%   Most of the actual work is done by the |GenerateNamedPkg| 
%   and\slash or |GeneratePkg| procedures.
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::modules_from_catalogue {target source args} {
   array set Opt {
      -formatpostamble {classical_postamble {##}}
      -formatpreamble  {classical_preamble {##}}
      -options         {}
      -postamble       {}
      -preamble        { }
      -sourceconf      {}
      -report          1
      -reportcmd       list
   }
   array set Opt $args
   variable filename $source
   variable fileoptions $Opt(-sourceconf)
   variable thefile [eval [list thefile $source] $fileoptions]
   variable Report {}  Report_store $Opt(-report) \
     Report_cmd $Opt(-reportcmd)
   set catalogue [extract $thefile\
     [linsert $Opt(-options) 0 docstrip.tcl::catalogue]\
     -metaprefix {#} -onerror puts]
   set c [interp create -safe]
   $c eval {
      proc unknown args {}
   }
   $c alias pkgProvide\
     [namespace which GenerateNamedPkg] $target\
     [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\
     [linsert $Opt(-formatpostamble) end $Opt(-postamble)]
   $c alias pkgIndex\
     [namespace which GeneratePkg] $target\
     [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\
     [linsert $Opt(-formatpostamble) end $Opt(-postamble)]
   $c alias fileoptions [namespace which fileoptions]
   $c eval $catalogue
   interp delete $c
   if {$Opt(-report)} then {return [join $Report \n]}
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{GenerateNamedPkg}
%   This procedure is an implementation of the |pkgProvide| catalogue 
%   command. The call syntax is
%   \begin{quote}
%     |GenerateNamedPkg| \word{target} \word{preamble-prefix} 
%     \word{postamble-prefix} \word{pkg-name} \word{version} 
%     \word{terminal-list}
%   \end{quote}
%   i.e., the alias should provide the first three arguments. 
%   \word{target} is the same as the \word{target} argument of 
%   |modules_from_catalogue|. The \word{preamble-prefix} and 
%   \word{postamble-prefix} arguments are command prefixes with the 
%   syntax
%   \begin{quote}
%     \meta{prefix} \word{target filename} \word{source filename} 
%     \word{terminal-list} 
%   \end{quote}
%   which will return the preamble and postamble texts respectively 
%   for the generated module file.
%   
%   The first part is extracting and handling extraction errors.
%   \begin{tcl}
proc docstrip::util::GenerateNamedPkg\
  {target preamblecmd postamblecmd name version terminals} {
   variable thefile
   if {[catch {
      extract $thefile $terminals -metaprefix {#}
   } text]} then {
      Report "Stripping error \"$text\"\nwhile indexing module\
        <[join $terminals ,]>."
   } else {
%   \end{tcl}
%   but after that it's all about generating the \texttt{.tm} file. 
%   Mapping |::| directly to |/| is a bit coarse, but it is what 
%   |::tcl::tm::UnknownHandler| does. Trimming away extra slashes 
%   protects against someone picking a package name beginning with 
%   |::|.
%   \begin{tcl}
      variable filename
      set module [format {%s-%s.tm}\
        [string trim [string map {:: /} $name] /] $version]
      set modL [file split $module]
      file mkdir [file join $target [file dirname $module]]
      set F [open [file join $target $module] w]
      fconfigure $F -encoding utf-8
      puts $F [eval $preamblecmd [list $module $filename $terminals]]
      puts -nonewline $F $text
      puts $F [eval $postamblecmd [list $module $filename $terminals]]
      close $F
      Report "Wrote $module"
   }
}
%   \end{tcl}
% \end{proc}
% 
% 
% \begin{proc}{GeneratePkg}
%   This procedure is an implementation of the |pkgIndex| catalogue 
%   command. It is basically the same as |GenerateNamedPkg|, but it 
%   must also (i)~index the extracted code to find out the package 
%   name and version, and (ii)~handle the case that the code declares 
%   several packages, by generating redirection files. 
%   The call syntax is
%   \begin{quote}
%     |GeneratePkg| \word{target} \word{preamble-prefix} 
%     \word{postamble-prefix} \word{terminal}\regstar
%   \end{quote}
%   i.e., the alias should provide the first three arguments. 
%   \word{target} is the same as the \word{target} argument of 
%   |modules_from_catalogue|. The \word{preamble-prefix} and 
%   \word{postamble-prefix} arguments are command prefixes with the 
%   syntax
%   \begin{quote}
%     \meta{prefix} \word{target filename} \word{source filename} 
%     \word{terminal-list} 
%   \end{quote}
%   which will return the preamble and postamble texts respectively 
%   for the generated module file.
%   
%   The first part is extracting and looking for package 
%   declarations, including the handling of errors during these 
%   operations.
%   \begin{tcl}
proc docstrip::util::GeneratePkg {target preamblecmd postamblecmd args} {
   variable thefile
   if {[catch {
      set text [extract $thefile $args -metaprefix {#}]
      packages_provided $text
   } res]} then {
      if {[lindex $::errorCode 0] eq "DOCSTRIP"} then {
         Report "Stripping error \"$res\"\nwhile indexing module\
           <[join $args ,]>."
      } else {
         Report "Code evaluation error:\n  $res\nwhile indexing\
           module <[join $args ,]>."
      }
%   \end{tcl}
%   There's also the corner case of not fining any package 
%   declaration,
%   \begin{tcl}
   } elseif {![llength $res]} then {
      Report "Found no package in module <[join $args ,]>."
   } else {
%   \end{tcl}
%   but after that it's all about generating \texttt{.tm} files. 
%   Mapping |::| directly to |/| is a bit coarse, but it is what 
%   |::tcl::tm::UnknownHandler| does. Trimming away extra slashes 
%   protects against someone picking a package name beginning with 
%   |::|.
%   \begin{tcl}
      variable filename
      set module [format {%s-%s.tm}\
        [string trim [string map {:: /} [lindex $res 0]] /]\
        [lindex $res 1]]
      set modL [file split $module]
      file mkdir [file join $target [file dirname $module]]
      set F [open [file join $target $module] w]
      fconfigure $F -encoding utf-8
      puts $F [eval $preamblecmd [list $module $filename $args]]
      puts -nonewline $F $text
      puts $F [eval $postamblecmd [list $module $filename $args]]
      close $F
      Report "Wrote $module"
%   \end{tcl}
%   Now, it might happen that a module provides more than package, 
%   and what should then be done for the extra packages? A reasonable 
%   solution seems to be to generate \texttt{.tm} files for all of 
%   them, but make the extra files consist of a single |source| 
%   command for the first file. Starting from the runtime 
%   |info script| value it is possible to compute the expected location 
%   of that file, but constructing the code to do it is a bit of work.
%   \begin{tcl}
      foreach {pkg ver} [lreplace $res 0 1] {
         set mod2 [format {%s-%s.tm}\
           [string trim [string map {:: /} $pkg] /] $ver]
         set mod2L [file split $mod2]
         file mkdir [file join $target [file dirname $mod2]]
         set common 0
         foreach d1 $modL d2 $mod2L {
            if {$d1 eq $d2} then {incr common} else {break}
         }
         set tail [lrange $modL $common end]
         set script {[::info script]}
         foreach d2 $mod2L {
            if {[incr common -1] < 0} then {
               set script "\[::file dirname $script\]"
            }
         }
         set F [open [file join $target $mod2] w]
         fconfigure $F -encoding utf-8
         puts $F "::source -encoding utf-8 \[::file join $script $tail\]"
         close $F
         Report "Wrote redirect $mod2"
      }
   }
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{classical_preamble}
%   This procedure generates preambles in the style of the \LaTeX\ 
%   \textsf{docstrip} utility. It has the call syntax
%   \begin{quote}
%     |docstrip::util::classical_preamble| \word{metaprefix}
%     \word{message} \word{target filename} 
%     \begin{regblock}[\regstar] \word{source filename} 
%     \word{terminal-list} \end{regblock}
%   \end{quote}
%   and returns the generated preamble.
%   
%   In comparison with \textsf{docstrip}, the \word{target filename} 
%   is |\outFileName|, the pairs of \word{source filename} and 
%   \word{terminal-list} are going to contribute to 
%   |\ReferenceLines|, and \word{message} is what gets added at the 
%   end.
%   \begin{tcl}
proc docstrip::util::classical_preamble {metaprefix message target args} {
   set res {""}
   lappend res " This is `$target',"
   lappend res { generated by the docstrip::util package.}
   lappend res {} { The original source files were:} {}
   foreach {source terminals} $args {
      set line " [file tail $source]"
      if {[llength $terminals]} then {
         append line { (with options: `} [join $terminals ,] {')}
      }
      lappend res $line
   }
   foreach line [split $message \n] {lappend res " $line"}
   return $metaprefix[join $res "\n$metaprefix"]
}
%</utilpkg>
%   \end{tcl}
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::classical_preamble] [arg metaprefix]\
  [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]]
  This command returns a preamble in the classical 
  [syscmd docstrip] style
[example {
##
## This is `TARGET',
## generated by the docstrip::util package.
##
## The original source files were:
##
## SOURCE (with options: `foo,bar')
## 
## Some message line 1
## line2
## line3
}]
  if called as
[example_begin]
docstrip::util::classical_preamble {##}\
  "\nSome message line 1\nline2\nline3" TARGET SOURCE {foo bar}
[example_end]
  The command supports preambles for files generated from multiple 
  sources, even though [cmd modules_from_catalogue] at present does 
  not need that.
%</utilman>
%    \end{macrocode}
% \end{proc}
% 
% \begin{proc}{classical_postamble}
%   This procedure generates postambles in the style of the \LaTeX\ 
%   \textsf{docstrip} utility. It has the call syntax
%   \begin{quote}
%     |docstrip::util::classical_postamble| \word{metaprefix}
%     \word{message} \word{target filename} 
%     \begin{regblock}[\regstar] \word{source filename} 
%     \word{terminal-list} \end{regblock}
%   \end{quote}
%   and returns the generated postamble.
%   
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::classical_postamble {metaprefix message target args} {
   set res {}
   foreach line [split $message \n] {lappend res " $line"}
   lappend res {} " End of file `$target'."
   return $metaprefix[join $res "\n$metaprefix"]
}
%</utilpkg>
%   \end{tcl}
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::classical_postamble] [arg metaprefix]\
  [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]]
  This command returns a postamble in the classical 
  [syscmd docstrip] style
[example {
## Some message line 1
## line2
## line3
##
## End of file `TARGET'.
}]
  if called as
[example_begin]
docstrip::util::classical_postamble {##}\
  "Some message line 1\nline2\nline3" TARGET SOURCE {foo bar}
[example_end]
  In other words, the [arg source] and [arg terminals] arguments are 
  ignored, but supported for symmetry with [cmd classical_preamble].
%</utilman>
%    \end{macrocode}
% \end{proc}
% 
% 
% \subsection{Scanning for declarations}
% 
% One task that must be performed is finding out which package(s) are 
% provided by a particular script (which will typically constitute 
% the contents of a module).
% 
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::packages_provided] [arg text]\
  [opt [arg setup-script]]]
  This command returns a list where every even index element is the 
  name of a package [cmd provide]d by [arg text] when that is 
  evaluated as a Tcl script, and the following odd index element is 
  the corresponding version. It is used to do package indexing of 
  extracted pieces of code, in the manner of [cmd pkg_mkIndex].
  [para]
  
  One difference to [cmd pkg_mkIndex] is that the [arg text] gets 
  evaluated in a safe interpreter. [cmd {package require}] commands 
  are silently ignored, as are unknown commands (which includes 
  [cmd source] and [cmd load]). Other errors cause 
  processing of the [arg text] to stop, in which case only those 
  package declarations that had been encountered before the error 
  will be included in the return value.
  [para]
  
  The [arg setup-script] argument can be used to customise the 
  evaluation environment, if the code in [arg text] has some very 
  special needs. The [arg setup-script] is evaluated in the local 
  context of the [cmd packages_provided] procedure just before the 
  [arg text] is processed. At that time, the name of the slave 
  command for the safe interpreter that will do this processing is 
  kept in the local variable [var c]. To for example copy the 
  contents of the [var ::env] array to the safe interpreter, one 
  might use a [arg setup-script] of
  [example {  $c eval [list array set env [array get ::env]]}]
%</utilman>
%    \end{macrocode}
% \begin{proc}{packages_provided}
%   This procedure looks for package declarations inside a script. It 
%   is derived from |pkg_mkIndex|, but simplified as it does not load 
%   binary libraries or invoke autoloading. The call syntax is
%   \begin{quote}
%     |packages_provided| \word{text} \word{setup-script}\regopt
%   \end{quote}
%   where \word{text} is the text to scan. The result is a list
%   \begin{quote}
%     \begin{regblock}[\regstar]\word{package} 
%     \word{version}\end{regblock}
%   \end{quote}
%   of packages that were declared.
%   
%   The \word{setup-script} is meant as a hook useful when indexing 
%   packages that have some special need. This argument is a script 
%   that gets evaluated (in the |packages_provided| procedure) 
%   \emph{after} the test interpreter used for loading packages in 
%   has been set up, but \emph{before} the text to index is evaluated 
%   in it. The local |c| variable holds the name of the interpreter 
%   command.
%   \changes{1.3}{2006/05/24}{Command added. (LH)}
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::packages_provided {text {setup ""}} {
%   \end{tcl}
%   First create the test interpreter and prepare it for use. Unlike 
%   the case in standard package indexing, this interpreter is safe 
%   (since safe interpreters are faster to create). Use the 
%   \word{setup-script} if you need to expose some unsafe feature.
%   \changes{1.3}{2010/03/28}{Using safe interpreter for package
%     indexing. (LH)}
%   \begin{tcl}
   set c [interp create -safe]
   $c eval {
      proc tclPkgUnknown args {}
      package unknown tclPkgUnknown
      proc unknown {args} {}
      proc auto_import {args} {}
   }
   $c hide package
   $c alias package [namespace which packages_provided,package] $c 
   eval $setup
%   \end{tcl}
%   Now evaluate the \word{text}. Errors are cheerfully ignored. Data 
%   for the return value is collected in the \describestring[local 
%   var.]{package_list}|package_list| local variable, which 
%   |packages_provided,package| uses |uplevel| to access.
%   \begin{tcl}
   set package_list {}
   catch {$c eval $text}
%   \end{tcl}
%   Cleanup and return the result.
%   \begin{tcl}
   interp delete $c
   return $package_list
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{packages_provided,package}
%   Calls to |package| in the test interpreter will be routed through 
%   this procedure, so that |provide|s can be seen and |request|s can 
%   be ignored. This is different from the mechanism in 
%   |pkg_mkIndex|, but I think this approach is more correct. The 
%   call syntax is
%   \begin{quote}
%     |packages_provided,package| \word{interp}
%     \word{subcommand} \word{argument}\regstar
%   \end{quote}
%   where \word{interp} is the slave interpreter command to use when 
%   actually carrying out the command. Remaining arguments are as for 
%   the core command |package|, which is assumed to be hidden in 
%   \word{interp}.
%   \begin{tcl}
proc docstrip::util::packages_provided,package {interp subcmd args} {
   switch -- $subcmd {
      r - re - req - requ - requi - requir - require {
         return 
      }
      pro - prov - provi - provid - provide {
         if {[llength $args] == 2} then {
            uplevel 1 [list lappend package_list] $args
         }
      }
   }
   eval [list $interp invokehidden package $subcmd] $args
}
%</utilpkg>
%   \end{tcl}
% \end{proc}
% 
%    \begin{macrocode}
%<*utilman>
[list_end]
%</utilman>
%    \end{macrocode}
% 
% 
% \section{Operations on source file text}
% 
%    \begin{macrocode}
%<*utilman>

[section {Source processing commands}]

Unlike the previous group of commands, which would use 
[cmd docstrip::extract] to extract some code lines and then process 
those further, the following commands operate on text consisting of 
all types of lines.

[list_begin definitions]
%</utilman>
%    \end{macrocode}
% 
% 
% 
% \subsection{Supporting doctools as markup language}
% 
% In the interest of making \textsf{docstrip} useful also for 
% programmers who do not want to write \LaTeX\ markup, some support is 
% offered also for files with \textsf{doctools} \texttt{.man} markup in 
% the comment lines. It is suggested that such files are given the 
% suffix \texttt{.ddt} to distinguish them from the \texttt{.dtx} files 
% that are directly \LaTeX able.
% 
% More precisely, it is suggested that the markup on comment and 
% metacomment lines of a \texttt{.ddt} file should follow the syntax on 
% the \texttt{doctools\_fmt} manpage~\cite{doctools_fmt}, or in the 
% future perhaps some derivative thereof. Unlike the case in 
% \texttt{.dtx} files, no explicit markup is required (or wanted) 
% around blocks of code and guard lines; such markup is to be generated 
% by the procedure below, as part of adding suitable markup to the code 
% lines.
% 
% \begin{proc}{ddt2man}
%   This procedure takes a string in the \texttt{.ddt} format sketched 
%   above and returns the corresponding text with \textsf{doctools} 
%   \texttt{.man} markup. The syntax is
%   \begin{quote}
%     |docstrip::util::ddt2man| \word{text}
%   \end{quote}
%   
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::ddt2man] [arg text]]
  The [cmd ddt2man] command reformats [arg text] from the general 
  [syscmd docstrip] format to [package doctools] [file .man] format 
  (Tcl Markup Language for Manpages). The different line types are 
  treated as follows:
  [list_begin definitions]
  [def {comment and metacomment lines}]
    The '%' and '%%' prefixes are removed, the rest of the text is 
    kept as it is.
  [def {empty lines}]
    These are kept as they are. (Effectively this means that they will 
    count as comment lines after a comment line and as code lines
    after a code line.)
  [def {code lines}]
    [cmd example_begin] and [cmd example_end] commands are placed 
    at the beginning and end of every block of consecutive code 
    lines. Brackets in a code line are converted to [cmd lb] and 
    [cmd rb] commands.
  [def {verbatim guards}]
    These are processed as usual, so they do not show up in the 
    result but every line in a verbatim block is treated as a code 
    line.
  [def {other guards}]
    These are treated as code lines, except that the actual guard is 
    [cmd emph]asised.
  [list_end]

  At the time of writing, no project has employed [package doctools] 
  markup in master source files, so experience of what works well is 
  not available. A source file could however look as follows
[example {
%</utilman>
%<*utilman,gcdexample>
%<<verbatim
% [manpage_begin gcd n 1.0]
% [keywords divisor]
% [keywords math]
% [moddesc {Greatest Common Divisor}]
% [require gcd [opt 1.0]]
% [description]
% 
% [list_begin definitions]
% [call [cmd gcd] [arg a] [arg b]]
%   The [cmd gcd] procedure takes two arguments [arg a] and [arg b] which 
%   must be integers and returns their greatest common divisor.
proc gcd {a b} {
%   The first step is to take the absolute values of the arguments.
%   This relieves us of having to worry about how signs will be treated 
%   by the remainder operation.
   set a [expr {abs($a)}]
   set b [expr {abs($b)}]
%   The next line does all of Euclid's algorithm! We can make do 
%   without a temporary variable, since $a is substituted before the 
%   [lb]set a $b[rb] and thus continues to hold a reference to the 
%   "old" value of [var a].
   while {$b>0} { set b [expr { $a % [set a $b] }] }
%   In Tcl 8.3 we might want to use [cmd set] instead of [cmd return]
%   to get the slight advantage of byte-compilation.
%<tcl83>  set a
%<!tcl83>   return $a
}
% [list_end]
%
% [manpage_end]
%verbatim
%</utilman,gcdexample>
%<*utilman>
}]
  If the above text is fed through [cmd docstrip::util::ddt2man] then 
  the result will be a syntactically correct [package doctools] 
  manpage, even though its purpose is a bit different.
  [para]
  
  It is suggested that master source code files with [package doctools] 
  markup are given the suffix [file .ddt], hence the "ddt" in 
  [cmd ddt2man].

%</utilman>
%    \end{macrocode}
% 
%   The structure of this procedure is fairly similar to that of 
%   |extract|, although of course the processing of the lines is rather 
%   different. The main novelty is the variable |wascode|, which is 
%   true if the previous line was a code line of some sort.
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::ddt2man {text} {
   set wascode 0
   set verbatim 0
   set res ""
   foreach line [split $text \n] {
      if {$verbatim} then {
         if {$line eq $endverbline} then {
            set verbatim 0
         } else {
            append res [string map {[ [lb] ] [rb]} $line] \n
         }
      } else {
         switch -glob -- $line %%* {
            if {$wacode} then {
               append res {[example_end]} \n
               set wascode 0
            }
            append res [string range $line 2 end] \n
         } %<<* {
            if {!$wascode} then {
               append res {[example_begin]} \n
               set wascode 1
            }
            set endverbline "%[string range $line 3 end]"
            set verbatim 1
         } %<* {
            if {!$wascode} then {
               append res {[example_begin]} \n
               set wascode 1
            }
            set guard ""
            regexp -- {(^%<[^>]*>)(.*)$} $line "" guard line
            append res \[ [list emph $guard] \]\
              [string map {[ [lb] ] [rb]} $line] \n
         } %* {
            if {$wascode} then {
               append res {[example_end]} \n
               set wascode 0
            }
            append res [string range $line 1 end] \n
         } {\\endinput} {
           break
         } "" {
%   \end{tcl}
%   Experience showed that empty lines at the beginning and end of a
%   file were hard to avoid. In order to stop those from being marked 
%   up as examples, an empty line will not trigger a switch to code 
%   mode.
%   \begin{tcl}
            append res \n
         } default {
            if {!$wascode} then {
               append res {[example_begin]} \n
               set wascode 1
            }
            append res [string map {[ [lb] ] [rb]} $line] \n
         }
      }
   }
   if {$wascode} then {append res {[example_end]} \n}
   return $res
}
%</utilpkg>
%   \end{tcl}
%   There is no test of this procedure, since it is rather 
%   experimental. One could however develop the example above into a 
%   test, if the need seems significant.
% \end{proc}
% 
% 
% \subsection{Guard information}
% 
% \begin{proc}{guards}
%   The |guards| command looks through a piece of master source code 
%   and gathers information about the guards occurring therein. The 
%   syntax is
%   \begin{quote}
%     |docstrip::util::guards| \word{subcommand} \word{text}
%   \end{quote}
%   where the \word{subcommand} is one of the following:
%   \begin{ttdescription}
%     \item[names]
%       Return the list of expression terminals occuring in the 
%       \word{text}, in no particular order.
%     \item[counts]
%       Return a dictionary which for each expression terminal 
%       occuring in the \word{text} gives the number of times it 
%       occurs.
%     \item[expressions]
%       Return the list of expressions occuring in the \word{text}, 
%       in no particular order.
%     \item[exprcounts]
%       Return a dictionary which for each guard expression occuring 
%       in the \word{text} gives the number of times it occurs.
%     \item[exprmods]
%       Return a dictionary which for each guard expression occuring 
%       in the \word{text} gives a string of the modifiers of these 
%       guards (where space is used for no modifier). This is the raw 
%       format of the information collected by this procedure.
%     \item[exprerr]
%       Return the list of syntactically incorrect expressions occuring 
%       in the \word{text}, in no particular order.
%     \item[rotten]
%       Return a dictionary which maps line numbers with bad guards to 
%       their contents.
%   \end{ttdescription}
%   \changes{1.1}{2005/03/02}{Command added. (LH, extending a 
%     suggestion of AK)}
%   \changes{1.2}{2005/08/23}{Changed name of \texttt{badguards} 
%     subcommand to \texttt{rotten}. (LH)}
%   \changes{1.2}{2005/08/26}{Changed name of \texttt{guard} 
%     procedure to \texttt{guards}. (LH)}
%   \begin{tcl}
%<*utilman>
[call [cmd docstrip::util::guards] [arg subcmd] [arg text]]
  The [cmd guards] command returns information (mostly of a 
  statistical nature) about the ordinary docstrip guards that occur 
  in the [arg text]. The [arg subcmd] selects what is returned.
  
  [list_begin definitions]
  [def [method counts]]
    List the guard expression terminals with counts. The format of 
    the return value is a dictionary which maps the terminal name to 
    the number of occurencies of it in the file.
  [def [method exprcount]]
    List the guard expressions with counts. The format of the return 
    value is a dictionary which maps the expression to the number of 
    occurencies of it in the file.
  [def [method exprerr]]
    List the syntactically incorrect guard expressions (e.g. 
    parentheses do not match, or a terminal is missing). The return 
    value is a list, with the elements in no particular order.
  [def [method expressions]]
    List the guard expressions. The return value is a list, with the 
    elements in no particular order.
  [def [method exprmods]]
    List the guard expressions with modifiers. The format of the return 
    value is a dictionary where each index is a guard expression and 
    each entry is a string with one character for every guard line that 
    has this expression. The characters in the entry specify what 
    modifier was used in that line: +, -, *, /, or (for guard without 
    modifier:) space. This is the most primitive form of the 
    information gathered by [cmd guards].
  [def [method names]]
    List the guard expression terminals. The return value is a list, 
    with the elements in no particular order.
  [def [method rotten]]
    List the malformed guard lines (this does not include lines where 
    only the expression is malformed, though). The format of the return 
    value is a dictionary which maps line numbers to their contents.
  [list_end]
%</utilman>
%   \end{tcl}
%   
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::guards {subcmd text} {
%   \end{tcl}
%   The first part is a cut-down |extract|. It collects data in the |E| 
%   array, which is indexed by expression. The |badL| variable is used 
%   for the data returned by the |rotten| subcommand.
%   \begin{tcl}
   set verbatim 0
   set lineno 1
   set badL {}
   foreach line [split $text \n] {
      if {$verbatim} then {
         if {$line eq $endverbline} then {set verbatim 0}
      } else {
         switch -glob -- $line %<<* {
            set endverbline "%[string range $line 3 end]"
            set verbatim 1
         } %<* {
            if {![
               regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\
                 modifier expression line
            ]} then {
               lappend badL $lineno $line
            } else {
               if {$modifier eq ""} then {set modifier " "}
               append E($expression) $modifier
            }
         }
      }
      incr lineno
   }
   if {$subcmd eq "rotten"} then {return $badL}
%   \end{tcl}
%   The second part processes the |E| array contents to produce the 
%   various subcommand results.
%   \begin{tcl}
   switch -- $subcmd "exprmods" {
      return [array get E]
   } "expressions" {
      return [array names E]
   } "exprerr" {
      set res {}
      foreach expr [array names E] {
         regsub -all {[^()!,|&]+} $expr 0 e
         regsub -all {,} $e {|} e
         if {[catch {expr $e}]} then {lappend res $expr}
      }
      return $res
   }
   foreach name [array names E] {
      set E($name) [string length $E($name)]
   }
   if {$subcmd eq "exprcounts"} then {return [array get E]}
   foreach expr [array names E] {
      foreach term [split $expr "()!,|&"] {
         if {$term eq ""} then {continue}
         if {![info exists T($term)]} then {set T($term) 0}
         incr T($term) $E($expr)
      }
   }
   switch -- $subcmd "counts" {
      return [array get T]
   } "names" {
      return [array names T]
   } default {
      error "Unknown subcommand '$subcmd', must be one of:\
        counts, exprcounts, expressions, exprmods, names, rotten"
   }
}
%</utilpkg>
%   \end{tcl}
% \end{proc}
% 
% 
% 
% \subsection{Backporting assistance}
% 
% It is (sadly) not entirely uncommon that the Literate Programmer finds 
% him- or herself with generated files that have been modified, even if 
% they carry prominent notices saying ``Don't do that! Change the 
% \emph{source} instead!''. When such changes are to the worse there is 
% little problem, because erasing them is just a matter of regenerating 
% the files in question, but often enough they instead contain useful 
% improvements of the code that one would like to keep. This requires 
% porting them back into the master source file, which in theory may 
% seem like a minor copy-and-paste task, but in practice often gets 
% frustrating because of the amount of navigating between the sites of 
% different changes that one must perform.
% 
% Ordinarily such backporting would be handled using patch files, and 
% that is what will be done also in this case, but the fact that the 
% file in which the modifications were made does not look like the 
% source file means traditional patching tools are not immediately 
% useful. The procedures defined below provides for 
% \textsc{docstrip}-aware patching.
% 
% 
% 
% \begin{proc}{patch}
%   The |patch| procedure applies a list of diff hunks to a 
%   \textsc{docstrip} style master source file. 
%   \changes{1.2}{2005/06/20}{Procedure added. (LH)}
%   The syntax is
%   \begin{quote}
%     |docstrip::util::patch| \word{source var.} \word{terminals} 
%     \word{fromtext} \word{diff}
%     \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock}
%   \end{quote}
%   The \word{source var.} is the name in the calling context 
%   of a variable which contains the list of lines in the source 
%   to patch; patching thus means modifying this list. \word{diff} is 
%   the difference hunks to apply, and the \word{fromtext} is the text 
%   that diff is meant to modify. \word{terminals} is the list of 
%   terminals one should use to |extract| \word{fromtext} (or a part 
%   thereof) from the source. The return value is a sort of annotated 
%   diff file, where each hunk carries a comment on how it was applied. 
%   Hunks with empty comments (usually meaning ``hunk applied in full, 
%   no problems were observed'') are omitted from this report.
%   
%   The \word{option} \word{value} pairs may be used to further control 
%   what happens. Currently the following options are interpreted:
%   \begin{ttdescription}
%     \item[-matching]
%       How is the \word{diff} matched against the \word{fromtext}? 
%       (Hunks that don't match are ignored.) The default is |exact|, 
%       which means each line must match. The alternatives are |none| 
%       (in which case no check is made, i.e., line numbers are silently 
%       assumed to be correct), |nonspace| (only non-whitespace 
%       characters are compared), and |anyspace| (any sequence of 
%       whitespace characters compare as a single space).
%     \item[-metaprefix]
%       Same as for |docstrip::extract|.
%     \item[-trimlines]
%       Same as for |docstrip::extract|.
%   \end{ttdescription}
%   
%   The \word{diff} is a list of ``parsed differences'', the format of 
%   which is explained in Subsection~\ref{Ssec:Tcldiff}.
%   
%   The way this procedure operates is that it first establishes a 
%   correspondence between lines in the source and lines in the 
%   \word{fromtext}. The first part of this correspondence is determined 
%   by the source and \word{terminals}, and is complicated but univocal. 
%   The second part of this correspondence is given by a matching of 
%   extracted lines to \word{fromtext} lines, and this is typically 
%   simple but not necessarily unique, which means the user need to be 
%   aware of the heuristic used: the lines of the \word{fromtext} are 
%   read in sequence, and whenever one matching the line after that in 
%   the extracted text with which the most recent correspondence is 
%   found, then these two are considered to correspond to each other. 
%   This should work well with the generated files one typically finds, 
%   which consist of long intervals of lines corresponding exactly to 
%   extracted texts, surrounded by some pre- and postambles. With files 
%   generated from several source files it may be necessary to add some 
%   metacomment line to disambigue the pieces, but that is often not a 
%   problem.
%   
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::patch {sourcevar termL fromtext diff args} {
   upvar 1 $sourcevar SL
   array set O {-trimlines 1 -matching exact}
   array set O $args
%   \end{tcl}
%   The first step is to construct the array |lift| that maps 
%   \word{fromtext} line numbers to source line numbers. This array 
%   actually contains a bit more than just the line numbers; the 
%   complete entry format is
%   \begin{quote}
%     \word{SL index} \word{source-prefix} \word{extract-prefix}
%   \end{quote}
%   where \word{SL index} is an index into the source \emph{list} of 
%   lines (thus starting at zero rather than one), \word{source-prefix} 
%   is the source line prefix (usually an empty string) that was removed 
%   as part of the extraction process, and \word{extract-prefix} is the 
%   prefix that was inserted as part of the extraction process (usually 
%   an empty string).
%   
%   In order to gather the above information, |extract| is run in the two 
%   lines of annotation format, which means the interpretation of an |EL| 
%   element depends heavily on its index modulo $3$. Setting the last 
%   element to a newline (it would otherwise had been an empty string, 
%   since |extract| places a newline after every line) is a sneaky way 
%   of preventing the |ptr| ``pointer'' into |EL| from going past the 
%   end of that list.
%   \begin{tcl}
   set cmd [list extract [join $SL \n]  $termL -annotate 2]
   foreach opt {-metaprefix -trimlines} {
      if {[info exists O($opt)]} then {lappend cmd $opt $O($opt)}
   }
   set EL [split [eval $cmd] \n]
   lset EL end \n
   set ptr 0
   set lineno 1
   set FL [list {}]
   foreach line [split $fromtext \n] {
      lappend FL $line
      if {$O(-trimlines)} then {set line [string trimright $line " "]}
      if {$line eq [lindex $EL $ptr]} then {
         set lift($lineno) [lindex $EL [incr ptr]]
         lset lift($lineno) 0 [expr { [lindex $EL [incr ptr]] - 1 }]
         incr ptr
      }
      incr lineno
   }
%   \end{tcl}
%   The |FL| variable constructed above is a list of \word{fromtext} 
%   lines, with list index equal to line number. It is used below when 
%   matching the differences.
%   
%   If at this point the |lift| array is empty, then no patching can be 
%   done. An error is thrown which suggests that the user checks the 
%   input given.
%   \begin{tcl}
   if {![array size lift]} then {
      return -code error "The extract did not match any part of the\
        fromtext. Check the list of terminals and the options"
   }
%   \end{tcl}
%   
%   The second step consists of extending the \word{diff} to a 
%   ``replace-list'', so that the hunk format becomes
%   \begin{quote}
%     \word{start1} \word{end1} \word{start2} \word{end2} 
%     \word{lines} \word{replaces}
%   \end{quote}
%   where the \word{replaces} is a list of lists on the form
%   \begin{quote}
%     \word{start1} \word{end1} \word{line}\regstar
%   \end{quote}
%   i.e., the same format as for arguments two and up of |lreplace|. (In 
%   particular, \(\mathit{end1} = \mathit{start1}-1\) when only 
%   inserting and there are no \word{line}s when only removing.) These 
%   extended hunks are placed in the variable |RL| sorted in descending 
%   order after \word{start1}, and the replaces within each hunk are 
%   sorted in that order too.
%   
%   This is also where the procedure begins constructing its report, 
%   which is another extension of the hunk format. Here the syntax is
%   \begin{quote}
%     \word{start1} \word{end1} \word{start2} \word{end2} 
%     \word{lines} \word{comment}
%   \end{quote}
%   where the \word{comment} is a description of what was done with 
%   this hunk.
%   \begin{tcl}
   set RL [list]
   set log [list]
   foreach hunk [lsort -decreasing -integer -index 0 $diff] {
      set replL [list]
      set l1 [lindex $hunk 0]
      set repl {0 -1}
      set matches 1
      foreach {type line} [lindex $hunk 4] {
         switch -glob -- $type {[0-]} {
            switch -- $O(-matching) "exact" {
               if {[lindex $FL $l1] ne $line} then {set matches 0}
            } "nonspace" {
               if {[regsub -all -- {\s} $line {}] ne\
                 [regsub -all -- {\s} [lindex $FL $l1] {}]} then {
                  set matches 0
               }
            } "anyspace" {
               if {[regsub -all -- {\s+} $line { }] ne\
                 [regsub -all -- {\s+} [lindex $FL $l1] { }]} then {
                  set matches 0
               }
            }
         }
         switch -- $type synch {
            if {[llength $repl]>2 ||\
              [lindex $repl 1]-[lindex $repl 0]>=0} then {
               lappend replL $repl
            }
            set repl [list $l1 [expr {$l1-1}]]
         } + {
            lappend repl $line
         } - {
            lset repl 1 $l1
            incr l1
         } 0 {
            if {[llength $repl]>2 ||\
              [lindex $repl 1]-[lindex $repl 0]>=0} then {
               lappend replL $repl
               set repl {0 -1}
            }
            lset repl 1 $l1
            incr l1
            lset repl 0 $l1
         }
      }
      if {[llength $repl]>2 || [lindex $repl 1]-[lindex $repl 0]>=0}\
      then {lappend replL $repl}
      if {$matches} then {
         lappend hunk [lsort -decreasing -integer -index 0 $replL]
         lappend RL $hunk
      } else {
         lappend hunk "(-- did not match fromtext --)"
         lappend log $hunk
      }
   }
%   \end{tcl}
%   The difference granularity is now the one that will be used in the 
%   insertion of new lines. The reason for extending hunks rather than 
%   using something else is to use the original data when reporting 
%   problems.
%   
%   The third step is to actually apply the changes to |SL|, translating 
%   line numbers as one goes along. Differences are processed 
%   back-to-front, because that means first file line numbers are valid, 
%   and those are the ones that can be translated to source line numbers.
%   
%   \begin{tcl}
   foreach hunk $RL {
      set applied 0
      set misapplied 0
%   \end{tcl}
%   For the purpose of generating a report, count is kept of how many 
%   lines of each hunk could be applied or could not be applied. That 
%   a hunk could not be applied (|applied| is zero) is often normal 
%   (the changed material was not generated from this source file), 
%   but if both counters are positive at the same time then one should 
%   take a bit more notice.
%   \begin{tcl}
      foreach repl [lindex $hunk 5] {
         unset -nocomplain from to
%   \end{tcl}
%   A |repl| is processed by replacing the item range |$from|--|$to| 
%   of lines to remove by the lines to insert, but for that the entire 
%   range of source lines must be continuous. The |for| loop below 
%   sets |from| and |to| to the endpoints of the first range of lines 
%   to replace because of this |repl|, and removes any subsequent 
%   ranges immediately.
%   \begin{tcl}
         for {set n [lindex $repl 1]} {$n>=[lindex $repl 0]}\
           {incr n -1} {
            if {![info exists lift($n)]} then {
               incr misapplied
               continue
            } elseif {![info exists from]} then {
               set to [lindex $lift($n) 0]
               set from $to
            } elseif {[lindex $lift($n) 0] == $from-1} then {
               set from [lindex $lift($n) 0]
            } else {
               set SL [lreplace $SL $from $to]
               set to [lindex $lift($n) 0]
               set from $to
            }
            incr applied
            set n0 $n
         }
%   \end{tcl}
%   For the replacement with lines to insert, it is necessary to 
%   figure out the source and extracted line prefixes. These are taken 
%   from the |from| line of the source, which is the line \emph{after} 
%   the new lines if this is a pure insertion.
%   \begin{tcl}
         if {[info exists from]} then {
            set sprefix [lindex $lift($n0) 1]
            set eprefix [lindex $lift($n0) 2]
         } elseif {[info exists lift([lindex $repl 0])]} then {
            foreach {from sprefix eprefix} $lift([lindex $repl 0])\ 
              break
            set to [expr {$from-1}]
         } else {
            incr misapplied [llength [lrange $repl 2 end]]
            continue
         }
         set eplen [string length $eprefix]
         set epend [expr {$eplen-1}]
%   \end{tcl}
%   Actually replacing the lines is pretty straightforward, but the 
%   |lreplace| command doing this is built dynamically.
%   \begin{tcl}
         set cmd [list lreplace $SL $from $to]
         foreach line [lrange $repl 2 end] {
            if {$eprefix eq [string range $line 0 $epend]} then {
               lappend cmd "$sprefix[string range $line $eplen end]"
            } else {
               lappend cmd $line
            }
            incr applied
         }
         set SL [eval $cmd]
      }
%   \end{tcl}
%   Only hunks with misapplied lines get included in the log.
%   \begin{tcl}
      if {$misapplied>0} then {
         if {$applied>0} then {
            lset hunk 5 "(-- was partially applied --)"
         } else {
            lset hunk 5 "(not applied)"
         }
         lappend log $hunk
      }
   }
%   \end{tcl}
%   Finally the log is formatted for return.
%   \begin{tcl}
   set res ""
   foreach hunk [lsort -index 0 -integer $log] {
      foreach {start1 end1 start2 end2 lines msg} $hunk break
      append res [format "@@ -%d,%d +%d,%d @@ %s\n"\
        $start1 [expr {$end1-$start1+1}]\
        $start2 [expr {$end2-$start2+1}] $msg]
      foreach {type line} $lines {
         switch -- $type 0 {
            append res " " $line \n
         } - - + {
            append res $type $line \n
         }
      }
   }
   return $res
}
%</utilpkg>
%   \end{tcl}
%   
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::patch] [arg source-var] [arg terminals]\
  [arg fromtext] [arg diff] [opt "[arg option] [arg value] ..."]]
  This command tries to apply a [syscmd diff] file (for example a 
  contributed patch) that was computed for a generated file to the 
  [syscmd docstrip] source. This can be useful if someone has 
  edited a generated file, thus mistaking it for being the source. 
  This command makes no presumptions which are specific for the case 
  that the generated file is a Tcl script.
  [para]
  
  [cmd patch] requires that the source file to patch is kept as a 
  list of lines in a variable, and the name of that variable in the 
  calling context is what goes into the [arg source-var] argument. 
  The [arg terminals] is the list of terminals used to extract the 
  file that has been patched. The [arg diff] is the actual diff to 
  apply (in a format as explained below) and the [arg fromtext] is 
  the contents of the file which served as "from" when the diff was 
  computed. Options can be used to further control the process.
  [para]
  
  The process works by "lifting" the hunks in the [arg diff] from 
  generated to source file, and then applying them to the elements of 
  the [arg source-var]. In order to do this lifting, it is necessary 
  to determine how lines in the [arg fromtext] correspond to elements 
  of the [arg source-var], and that is where the [arg terminals] come 
  in; the source is first [cmd extract]ed under the given 
  [arg terminals], and the result of that is then matched against 
  the [arg fromtext]. This produces a map which translates line 
  numbers stated in the [arg diff] to element numbers in 
  [arg source-var], which is what is needed to lift the hunks.
  [para]
  
  The reason that both the [arg terminals] and the [arg fromtext] 
  must be given is twofold. First, it is very difficult to keep track 
  of how many lines of preamble are supplied some other way than by 
  copying lines from source files. Second, a generated file might 
  contain material from several source files. Both make it impossible 
  to predict what line number an extracted file would have in the 
  generated file, so instead the algorithm for computing the line 
  number map looks for a block of lines in the [arg fromtext] which 
  matches what can be extracted from the source. This matching is 
  affected by the following options: 
  [list_begin options]
  [opt_def -matching [arg mode]]
    How equal must two lines be in order to match? The supported 
    [arg mode]s are:
    [list_begin definitions]
    [def [const exact]]
      Lines must be equal as strings. This is the default.
    [def [const anyspace]]
      All sequences of whitespace characters are converted to single 
      spaces before comparing.
    [def [const nonspace]]
      Only non-whitespace characters are considered when comparing.
    [def [const none]]
      Any two lines are considered to be equal.
    [list_end]
  [opt_def -metaprefix [arg string]]
    The [option -metaprefix] value to use when extracting. Defaults 
    to "%%", but for Tcl code it is more likely that "#" or "##" had 
    been used for the generated file.
  [opt_def -trimlines [arg boolean]]
    The [option -trimlines] value to use when extracting. Defaults to 
    true.
  [list_end]
  
  The return value is in the form of a unified diff, containing only 
  those hunks which were not applied or were only partially applied; 
  a comment in the header of each hunk specifies which case is at 
  hand. It is normally necessary to manually review both the return 
  value from [cmd patch] and the patched text itself, as this command 
  cannot adjust comment lines to match new content.
  [para]
  
  An example use would look like
[example_begin]
set sourceL [lb]split [lb]docstrip::util::thefile from.dtx[rb] \n[rb]
set terminals {foo bar baz}
set fromtext [lb]docstrip::util::thefile from.tcl[rb]
set difftext [lb]exec diff --unified from.tcl to.tcl[rb]
set leftover [lb]docstrip::util::patch sourceL $terminals $fromtext\
  [lb]docstrip::util::import_unidiff $difftext[rb] -metaprefix {#}[rb]
set F [lb]open to.dtx w[rb]; puts $F [lb]join $sourceL \n[rb]; close $F
return $leftover
[example_end]
  Here, [file from.dtx] was used as source for [file from.tcl], which 
  someone modified into [file to.tcl]. We're trying to construct a 
  [file to.dtx] which can be used as source for [file to.tcl].
%</utilman>
%    \end{macrocode}
% \end{proc}
% 
% 
% 
% 
% \section{Reading files}
% 
% \subsection{Raw file contents}
% 
% \begin{proc}{thefile}
%   When experimenting with docstripping, it is often convenient to have 
%   an easy command for reading the contents of a file. The |thefile| 
%   command (named vaugely in the tradition of such \LaTeX\ commands as 
%   |\thepage|) returns the contents of the file whose name it is given. 
%   \changes{1.2}{2005/06/19}{Procedure added. (LH)}
%   More precisely, the syntax is
%   \begin{quote}
%     |docstrip::util::thefile| \word{file name} 
%     \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock}
%   \end{quote}
%   where the \word{option} \word{value} pairs are handed to |fconfigure| 
%   to configure the file before reading it.
%   \changes{1.2}{2005/09/07}{Added error handling. (LH)}
%   \changes{1.3}{2010/04/12}{Added \texttt{-nonewline} switch. (LH)}
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::thefile {fname args} {
   set F [open $fname r]
   if {[llength $args]} then {
      if {[set code [
         catch {eval [linsert $args 0 fconfigure $F]} res
      ]]} then {
         close $F
         return -code $code -errorinfo $::errorInfo -errorcode\
           $::errorCode
      }
   }
   catch {read -nonewline $F} res
   close $F
   return $res
}
%</utilpkg>
%   \end{tcl}
%   The code is thus very straightforward---what remains is to make a 
%   manpage entry for it.
%   \begin{tcl}
%<*utilman>
[call [cmd docstrip::util::thefile] [arg filename] [
  opt "[arg option] [arg value] ..."
]]
  The [cmd thefile] command opens the file [arg filename], reads it to 
  end, closes it, and returns the contents (dropping a final newline 
  if there is one). The option-value pairs are 
  passed on to [cmd fconfigure] to configure the open file channel 
  before anything is read from it.
%</utilman>
%   \end{tcl}
% \end{proc}
% 
% Better provide some tests too\dots
% \begin{tcl}
%<*utiltest>
tcltest::test docstrip::util::thefile-1.1 {thefile without args}\
  -setup {
   set Fname [tcltest::makeFile [
      join {
         {% Just a minor test file.}
         {puts A}
         {%<*bar>}
         {puts B}
         {%<*foo>}
         {puts [info exists baz]}
      } \n
   ] test.txt]
} -body {
   docstrip::util::thefile $Fname
} -cleanup {
   tcltest::removeFile $Fname
} -result [join {
   {% Just a minor test file.}
   {puts A}
   {%<*bar>}
   {puts B}
   {%<*foo>}
   {puts [info exists baz]}
} \n]
% \end{tcl}
% This one tests that an error in the number of arguments is caught 
% correctly.
% \begin{tcl}
tcltest::test docstrip::util::thefile-1.2 {thefile with wrong no. args}\
  -setup {
   set Fname [tcltest::makeFile [
      join {
         {% Just a minor test file (contents irrelevant).}
         {puts A}
         {%<*bar>}
         {puts B}
         {%<*foo>}
         {puts [info exists baz]}
      } \n
   ] test.txt]
} -body {
   docstrip::util::thefile $Fname -translation binary -buffering
} -cleanup {
   tcltest::removeFile $Fname
} -returnCodes error
% \end{tcl}
% This one tests configuring (of encoding).
% \begin{tcl}
tcltest::test docstrip::util::thefile-1.3 {thefile with args} -setup {
   set Fname [tcltest::makeFile "Dummy content to overwrite" test.xxx]
   set F [open $Fname w]
   fconfigure $F -translation binary
   puts -nonewline $F [encoding convertto utf-8 \u00E5\u00E4\u00F6]
   close $F
} -body {
   docstrip::util::thefile $Fname -encoding utf-8
} -cleanup {
   tcltest::removeFile $Fname
} -result \u00E5\u00E4\u00F6
%</utiltest>
% \end{tcl}
% 
% 
% \subsection{The diff format}
% \label{Ssec:Tcldiff}
% 
% The difference format used by |docstrip::util::patch| is a 
% \Tcllogo-list format into which the common diff formats can be parsed. 
% Each hunk is a five element list
% \begin{quote}
%   \word{start1} \word{end1} \word{start2} \word{end2} \word{lines}
% \end{quote}
% where the start and end elements are integers, specifying the line 
% numbers of the first and last lines in the hunk respectively. 1 
% elements pertain to the first file and 2 elements to the second file. 
% The number of the first line in a file is |1|.
% 
% The \word{lines} is a list of the form
% \begin{quote}
%   \begin{regblock}[\regplus]\word{type} \word{text}\end{regblock}
% \end{quote}
% where each \word{text} is an actual line of text (minus newline) and 
% the \word{type} specifies the type of this line. |+| means a line that 
% is in the second file but not the first, |-| means a line that is in 
% the first file but not the second, and |0| means a line that is in 
% both files. The format is thus most similar to the \texttt{--unified} 
% diff format, but the difference is not too great to the other formats 
% either. The \word{type} may also be |synch|, which is used as a 
% placeholder for any number of ``invisible'' lines (neither in the 
% first or second file, but perhaps present in the source) at that 
% point. The \word{text} of |synch| lines is ignored.
% 
% As an example of what it looks like, a difference between the two 
% files
%\begin{verbatim}
%foo
%bar baz
%end
%\end{verbatim}
% and 
%\begin{verbatim}
%foo
%bar
%baz
%end
%\end{verbatim}
% is
% \begin{quote}
%   |1 3 1 4 {0 foo - {bar baz} + bar + baz 0 end}|
% \end{quote}
% An alternative is
% \begin{quote}
%   |2 2 2 3 {+ bar - {bar baz} + baz}|
% \end{quote}
% since \texttt{+} lines ``commute'' with \texttt{-} lines. 
% 
% \begin{proc}{import_unidiff}
%   The |import_unidiff| procedure imports a standard diff in unified 
%   format to the format described above. The call syntax is
%   \begin{quote}
%     |docstrip::util::import_unidiff| \word{diff-text} 
%     \word{warning-var}\regopt
%   \end{quote}
%   where the \word{diff-text} is the actual text of the diff file to 
%   convert, and the \word{warning-var} is the name of a variable in 
%   the calling context to which warnings about failings to parse the 
%   input \word{diff-text} will be appended. The return format is a 
%   list as described above.
%   
%   In the implementation, the hunk-to-be is kept in the five 
%   variables |start1|, |end1|, |start2|, |end2|, and |lines|. 
%   Whether |end2| is an integer is used as a signal for whether there 
%   is a hunk to append. Malformed hunk headers will cause that hunk 
%   to be ignored.
%   \begin{tcl}
%<*utilpkg>
proc docstrip::util::import_unidiff {text {warnvar ""}} {
   if {$warnvar ne ""} then {upvar 1 $warnvar warning}
   set inheader 1
   set res [list]
   set lines [list]
   set end2 "not an integer"
   foreach line [split $text \n] {
      if {$inheader && [regexp {^(---|\+\+\+)} $line]}\
      then {continue}
      switch -glob -- $line { *} {
         lappend lines 0 [string range $line 1 end]
      } {+*} {
         lappend lines + [string range $line 1 end]
      } {-*} {
         lappend lines - [string range $line 1 end]
      } @@* {
         if {[string is integer $end2]} then {
            lappend res [list $start1 $end1 $start2 $end2 $lines]
         }
         set len2 [set len1 ,1]
         if {[
            regexp {^@@ -([0-9]+)(,[0-9]+)? \+([0-9]+)(,[0-9]+)? @@}\
              $line -> start1 len1 start2 len2
         ] && [scan "$start1 $len1,1" {%d ,%d} start1 len1]==2 &&\
              [scan "$start2 $len2,1" {%d ,%d} start2 len2]==2
         } then {
            set end1 [expr {$start1+$len1-1}]
            set end2 [expr {$start2+$len2-1}]
            set inheader 0
         } else {
            set end2 "not an integer"
            append warning "Could not parse hunk header:  " $line \n
         }
         set lines [list]
      } "" {
%   \end{tcl}
%   Empty lines are ignored (there will typically be one at the end of 
%   the |foreach| loop).
%   \begin{tcl}
      } default {
         append warning "Could not parse line:  " $line \n
      }
   }
   if {[string is integer $end2]} then {
      lappend res [list $start1 $end1 $start2 $end2 $lines]
   }
   return $res
}
%</utilpkg>
%   \end{tcl}
%   
%    \begin{macrocode}
%<*utilman>
[call [cmd docstrip::util::import_unidiff] [arg diff-text]\
  [opt [arg warning-var]]]
  This command parses a unified ([syscmd diff] flags [option -U] and 
  [option --unified]) format diff into the list-of-hunks format 
  expected by [cmd docstrip::util::patch]. The [arg diff-text] 
  argument is the text to parse and the [arg warning-var] is, if 
  specified, the name in the calling context of a variable to which 
  any warnings about parsing problems will be [cmd append]ed.
  [para]
  
  The return value is a list of [term hunks]. Each hunk is a list of 
  five elements "[arg start1] [arg end1] [arg start2] [arg end2] 
  [arg lines]". [arg start1] and [arg end1] are line numbers in the 
  "from" file of the first and last respectively lines of the hunk. 
  [arg start2] and [arg end2] are the corresponding line numbers in 
  the "to" file. Line numbers start at 1. The [arg lines] is a list 
  with two elements for each line in the hunk; the first specifies the 
  type of a line and the second is the actual line contents. The type 
  is [const -] for lines only in the "from" file, [const +] for lines 
  that are only in the "to" file, and [const 0] for lines that are 
  in both.
[list_end]
%</utilman>
%    \end{macrocode}
% \end{proc}
% 
% 
% 
% \section{Closing material}
% 
% The packages need no particular ending, but the tests can do with an
% explicit cleanup.
% 
% \begin{tcl}
%<*test,utiltest>
%<!tcllibtest>tcltest::cleanupTests
%<tcllibtest>testsuiteCleanup
%</test,utiltest>
% \end{tcl}
% 
% The manpages require an explicit ending, and can do with some 
% keywords.
%    \begin{macrocode}
%<*man,utilman>
[manpage_end]
%</man,utilman>
%    \end{macrocode}
% There! That's it!
% 
% 
% \section{Development tools}
% 
% I have found the following code snippets useful for formatting 
% \texttt{docstrip.man}.
% \begin{tcl}
%<*devmantest>
package require doctools
doctools::new man2html -format html
proc makehtml {{from docstrip.man} {to docstrip.html}} {
   set text [string map {\r \n}\
     [getText -w $from [minPos] [maxPos -w $from]]]
   set html [man2html format $text]
   replaceText -w $to [minPos] [maxPos -w $to]\
     [string map {\n \r} $html]
}
proc dtx2html {terminals {to docstrip_util.html} {from tcldocstrip.dtx}} {
   set text [string map {\r \n}\
     [getText -w $from [minPos] [maxPos -w $from]]]
   set html [man2html format [docstrip::extract $text $terminals]]
   replaceText -w $to [minPos] [maxPos -w $to]\
     [string map {\n \r} $html]
}
%</devmantest>
% \end{tcl}
% It is included here so that I know where to find it, but it is
% normally no extracted.
% 
% \bigskip
% 
% The following block of code could be taken as the beginnings of a test 
% or example of the use of |ddt2man|. First extract the 
% \Module{gcdexample}.
% \begin{tcl}
%<*devtest2>
package require docstrip
set F [open tcldocstrip.dtx r]
set text [docstrip::extract [read $F] gcdexample]
close $F
% \end{tcl}
% Then unindent the lines so that they become the intended mixture of 
% code and comment lines.
% \begin{tcl}
regsub -all -lineanchor {^   } $text "" ddt
% \end{tcl}
% Now |ddt2html| can be applied:
% \begin{tcl}
package require docstrip::util
set man [docstrip::util::ddt2man $ddt]
% \end{tcl}
% Finally, format this code as something.
% \begin{tcl}
package require doctools
doctools::new man2html -format html
set html [man2html format $man]
%</devtest2>
% \end{tcl}
% 
% \begin{thebibliography}{6}
% \bibitem{tclldoc}
%   Lars Hellstr\"om:
%   \textit{The \textsf{tclldoc} package and class}, 
%   \LaTeXe\ package and document class,
%   \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash 
%   \texttt{latex}\slash \texttt{contrib}\slash \texttt{tclldoc}/.
% \bibitem{doctools_fmt}
%   Andreas Kupries:
%   \textit{Specification of a simple \Tcllogo\ Markup Language 
%   for Manpages}, manpage,
%   \texttt{tcllib} module \textsf{doctools}, 2002--;
%   \textsc{http}:/\slash \texttt{core.tcl.tk/tcllib}\slash
%   \texttt{doc}\slash \texttt{doctools\_fmt.html}.
% \bibitem{docstrip}
%   Frank Mittelbach, Denys Duchier, Johannes Braams, Marcin 
%   Woli\'nski, and Mark Wooding: \textit{The \textsf{DocStrip} 
%   program},  The \LaTeX3 Project; 
%   \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash 
%   \texttt{latex}\slash \texttt{base}\slash \texttt{docstrip.dtx}.
% \bibitem{doc}
%   Frank Mittelbach, B.~Hamilton Kelly, Andrew Mills, Dave Love, and 
%   Joachim \mbox{Schrod}: \textit{The \textsf{doc} and 
%   \textsf{shortvrb} Packages}, The \LaTeX3 Project;
%   \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash 
%   \texttt{latex}\slash \texttt{base}\slash \texttt{doc.dtx}.
% \iffalse
% [enum]
%   Chapter 14 of
%   [emph {The LaTeX Companion}] (second edition),
%   Addison-Wesley, 2004; ISBN 0-201-36299-6.
% \fi
% \end{thebibliography}
% 
% 
\endinput