diff options
author | William Joye <wjoye@cfa.harvard.edu> | 2016-10-27 19:39:39 (GMT) |
---|---|---|
committer | William Joye <wjoye@cfa.harvard.edu> | 2016-10-27 19:39:39 (GMT) |
commit | ea28451286d3ea4a772fa174483f9a7a66bb1ab3 (patch) | |
tree | 6ee9d8a7848333a7ceeee3b13d492e40225f8b86 /tcllib/modules/docstrip | |
parent | b5ca09bae0d6a1edce939eea03594dd56383f2c8 (diff) | |
parent | 7c621da28f07e449ad90c387344f07a453927569 (diff) | |
download | blt-ea28451286d3ea4a772fa174483f9a7a66bb1ab3.zip blt-ea28451286d3ea4a772fa174483f9a7a66bb1ab3.tar.gz blt-ea28451286d3ea4a772fa174483f9a7a66bb1ab3.tar.bz2 |
Merge commit '7c621da28f07e449ad90c387344f07a453927569' as 'tcllib'
Diffstat (limited to 'tcllib/modules/docstrip')
-rw-r--r-- | tcllib/modules/docstrip/ChangeLog | 127 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip.man | 435 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip.tcl | 163 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip.test | 243 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip_util.man | 586 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip_util.tcl | 649 | ||||
-rw-r--r-- | tcllib/modules/docstrip/docstrip_util.test | 84 | ||||
-rw-r--r-- | tcllib/modules/docstrip/pkgIndex.tcl | 23 | ||||
-rw-r--r-- | tcllib/modules/docstrip/tcldocstrip.dtx | 4012 | ||||
-rw-r--r-- | tcllib/modules/docstrip/tcldocstrip.ins | 46 | ||||
-rw-r--r-- | tcllib/modules/docstrip/tcldocstrip.stitch | 25 |
11 files changed, 6393 insertions, 0 deletions
diff --git a/tcllib/modules/docstrip/ChangeLog b/tcllib/modules/docstrip/ChangeLog new file mode 100644 index 0000000..bcc9767 --- /dev/null +++ b/tcllib/modules/docstrip/ChangeLog @@ -0,0 +1,127 @@ +2013-02-01 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.15 ======================== + * + +2011-12-13 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.14 ======================== + * + +2011-01-24 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.13 ======================== + * + +2010-09-13 Lars Hellstr\"om <lars_h@users.sourceforge.net> + + docstrip::util version is now 1.3. + + * New docstrip::util feature: + In-file catalogue of file contents, + which can be used to create pkgIndex.tcl + entries or .tm files. + * Improved documentation of existing + docstrip::util commands. + * Changed docstrip::util::thefile to strip + away a final newline. + * Fixed bug #3036841. + +2009-12-07 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.12 ======================== + * + +2009-01-28 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * tcldocstrip.dtx: Moved a number of documentation cleanup changes + * docstrip.man: into the master DTX file, and added category + * docstrip_util.man: information. + +2008-12-12 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.11.1 ======================== + * + +2008-10-16 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.11 ======================== + * + +2007-09-12 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.10 ======================== + * + +2007-03-28 Andreas Kupries <andreask@activestate.com> + + * apps/tcldocstrip: Added a block of meta data. + +2006-10-03 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.9 ======================== + * + +2006-09-13 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * tcldocstrip.dtx: Modified the setup of the testsuite to match + the other modules and packages in tcllib. The testsuite + especially now handles execution in a too old a Tcl core + properly. + * docstrip.test: Regenerated. + * docstrip_util.test: Regenerated. + +2005-10-06 Andreas Kupries <andreas_kupries@users.sourceforge.net> + + * + * Released and tagged Tcllib 1.8 ======================== + * + +2005-10-04 Andreas Kupries <andreask@activestate.com> + + * tcldocstrip.dtx: Applied changes made by Lars to fix + * tcldocstrip.ins: a number of bugs he found. I am doing + * tcldocstrip.stitch: it in his stead as he currently has + trouble with the SF CVS. Also fixed a syntax error in the + documentation. + + * Regenerated the other files. One new file, + "docstrip_util.test". All tests pass, regular and from within + the test harness. + +2005-09-26 Andreas Kupries <andreask@activestate.com> + + * tcldocstrip.dtx: Fixed the testsuite bug regarding access to + files in the module under test. + + * docstrip.test: Regenerated. + +2005-08-30 Andreas Kupries <andreask@activestate.com> + + * tcldocstrip.dtx: + * docstrip_util.man: Fixed formatting problem in manpage. (Added a + missing closing bracket, and removed bad splitting across lines). + + * Added entry for a large commit done by Lars to the ChangeLog, on + behalf on Lars. See entry immediately below. + +2005-08-28 Lars Hellstroem + + * New docstrip::util commands: guards, thefile, patch, and + import_unidiff. New -annotate option of docstrip::extract (used + by docstrip::util::patch). patch and import_unidiff still lack + .man documentation. The docstrip::util package still lacks + tests. + +2005-02-14 Andreas Kupries <andreask@activestate.com> + + * docstrip: New module, by Lars Hellstroem, to support literate programming. + diff --git a/tcllib/modules/docstrip/docstrip.man b/tcllib/modules/docstrip/docstrip.man new file mode 100644 index 0000000..95795d4 --- /dev/null +++ b/tcllib/modules/docstrip/docstrip.man @@ -0,0 +1,435 @@ +[manpage_begin docstrip n 1.2] +[see_also docstrip_util] +[keywords .dtx] +[keywords docstrip] +[keywords documentation] +[keywords LaTeX] +[keywords {literate programming}] +[keywords source] +[copyright "2003\u20132010 Lars Hellstr\u00F6m\ + <Lars dot Hellstrom at residenset dot net>"] +[moddesc {Literate programming tool}] +[titledesc {Docstrip style source code extraction}] +[category {Documentation tools}] +[require Tcl 8.4] +[require docstrip [opt 1.2]] +[vset emdash \u2014] +[description] + +[syscmd Docstrip] is a tool created to support a brand of Literate +Programming. It is most common in the (La)TeX community, where it +is being used for pretty much everything from the LaTeX core and up, +but there is nothing about [syscmd docstrip] which prevents using it +for other types of software. +[para] + +In short, the basic principle of literate programming is that program +source should primarily be written and structured to suit the +developers (and advanced users who want to peek "under the hood"), not +to suit the whims of a compiler or corresponding source code consumer. +This means literate sources often need some kind of "translation" to an +illiterate form that dumb software can understand. +The [package docstrip] Tcl package handles this translation. +[para] + +Even for those who do not whole-hartedly subscribe to the philosophy +behind literate programming, [syscmd docstrip] can bring greater +clarity to in particular: +[list_begin itemized] + [item] programs employing non-obvious mathematics + [item] projects where separate pieces of code, perhaps in + different languages, need to be closely coordinated. +[list_end] +The first is by providing access to much more powerful typographical +features for source code comments than are possible in plain text. +The second is because all the separate pieces of code can be kept +next to each other in the same source file. +[para] + +The way it works is that the programmer edits directly only one or +several "master" source code files, from which [syscmd docstrip] +generates the more traditional "source" files compilers or the like +would expect. The master sources typically contain a large amount of +documentation of the code, sometimes even in places where the code +consumers would not allow any comments. The etymology of "docstrip" +is that this [emph doc]umentation was [emph strip]ped away (although +"code extraction" might be a better description, as it has always +been a matter of copying selected pieces of the master source rather +than deleting text from it). +The [package docstrip] Tcl package contains a reimplementation of +the basic extraction functionality from the [syscmd docstrip] +program, and thus makes it possible for a Tcl interpreter to read +and interpret the master source files directly. +[para] + +Readers who are not previously familiar with [syscmd docstrip] but +want to know more about it may consult the following sources. +[list_begin enumerated] +[enum] + [emph {The tclldoc package and class}], + [uri {http://ctan.org/tex-archive/macros/latex/contrib/tclldoc/}]. +[enum] + [emph {The DocStrip utility}], + [uri {http://ctan.org/tex-archive/macros/latex/base/docstrip.dtx}]. +[enum] + [emph {The doc and shortvrb Packages}], + [uri {http://ctan.org/tex-archive/macros/latex/base/doc.dtx}]. +[enum] + Chapter 14 of + [emph {The LaTeX Companion}] (second edition), + Addison-Wesley, 2004; ISBN 0-201-36299-6. +[list_end] + +[section {File format}] + +The basic unit [syscmd docstrip] operates on are the [emph lines] of +a master source file. Extraction consists of selecting some of these +lines to be copied from input text to output text. The basic +distinction is that between [emph {code lines}] (which are copied and +do not begin with a percent character) and [emph {comment lines}] +(which begin with a percent character and are not copied). + +[example { + docstrip::extract [join { + {% comment} + {% more comment !"#$%&/(} + {some command} + { % blah $blah "Not a comment."} + {% abc; this is comment} + {# def; this is code} + {ghi} + {% jkl} + } \n] {} +}] +returns the same sequence of lines as +[example { + join { + {some command} + { % blah $blah "Not a comment."} + {# def; this is code} + {ghi} "" + } \n +}] + +It does not matter to [syscmd docstrip] what format is used for the +documentation in the comment lines, but in order to do better than +plain text comments, one typically uses some markup language. Most +commonly LaTeX is used, as that is a very established standard and +also provides the best support for mathematical formulae, but the +[package docstrip::util] package also gives some support for +[term doctools]-like markup. +[para] + +Besides the basic code and comment lines, there are also +[emph {guard lines}], which begin with the two characters '%<', and +[emph {meta-comment lines}], which begin with the two characters +'%%'. Within guard lines there is furthermore the distinction between +[emph {verbatim guard lines}], which begin with '%<<', and ordinary +guard lines, where the '%<' is not followed by another '<'. The last +category is by far the most common. +[para] + +Ordinary guard lines conditions extraction of the code line(s) they +guard by the value of a boolean expression; the guarded block of +code lines will only be included if the expression evaluates to true. +The syntax of an ordinary guard line is one of +[example { + '%' '<' STARSLASH EXPRESSION '>' + '%' '<' PLUSMINUS EXPRESSION '>' CODE +}] +where +[example { + STARSLASH ::= '*' | '/' + PLUSMINUS ::= | '+' | '-' + EXPRESSION ::= SECONDARY | SECONDARY ',' EXPRESSION + | SECONDARY '|' EXPRESSION + SECONDARY ::= PRIMARY | PRIMARY '&' SECONDARY + PRIMARY ::= TERMINAL | '!' PRIMARY | '(' EXPRESSION ')' + CODE ::= { any character except end-of-line } +}] +Comma and vertical bar both denote 'or'. Ampersand denotes 'and'. +Exclamation mark denotes 'not'. A TERMINAL can be any nonempty string +of characters not containing '>', '&', '|', comma, '(', or ')', +although the [syscmd docstrip] manual is a bit restrictive and only +guarantees proper operation for strings of letters (although even +the LaTeX core sources make heavy use also of digits in TERMINALs). +The second argument of [cmd docstrip::extract] is the list of those +TERMINALs that should count as having the value 'true'; all other +TERMINALs count as being 'false' when guard expressions are evaluated. +[para] + +In the case of a '%<*[emph EXPRESSION]>' guard, the lines guarded are +all lines up to the next '%</[emph EXPRESSION]>' guard with the same +[emph EXPRESSION] (compared as strings). The blocks of code delimited +by such '*' and '/' guard lines must be properly nested. +[example { + set text [join { + {begin} + {%<*foo>} + {1} + {%<*bar>} + {2} + {%</bar>} + {%<*!bar>} + {3} + {%</!bar>} + {4} + {%</foo>} + {5} + {%<*bar>} + {6} + {%</bar>} + {end} + } \n] + set res [docstrip::extract $text foo] + append res [docstrip::extract $text {foo bar}] + append res [docstrip::extract $text bar] +}] +sets $res to the result of +[example { + join { + {begin} + {1} + {3} + {4} + {5} + {end} + {begin} + {1} + {2} + {4} + {5} + {6} + {end} + {begin} + {5} + {6} + {end} "" + } \n +}] + +In guard lines without a '*', '/', '+', or '-' modifier after the +'%<', the guard applies only to the CODE following the '>' on that +single line. A '+' modifier is equivalent to no modifier. A '-' +modifier is like the case with no modifier, but the expression is +implicitly negated, i.e., the CODE of a '%<-' guard line is only +included if the expression evaluates to false. +[para] + +Metacomment lines are "comment lines which should not be stripped +away", but be extracted like code lines; these are sometimes used for +copyright notices and similar material. The '%%' prefix is however +not kept, but substituted by the current [option -metaprefix], which +is customarily set to some "comment until end of line" character (or +character sequence) of the language of the code being extracted. +[example { + set text [join { + {begin} + {%<foo> foo} + {%<+foo>plusfoo} + {%<-foo>minusfoo} + {middle} + {%% some metacomment} + {%<*foo>} + {%%another metacomment} + {%</foo>} + {end} + } \n] + set res [docstrip::extract $text foo -metaprefix {# }] + append res [docstrip::extract $text bar -metaprefix {#}] +}] +sets $res to the result of +[example { + join { + {begin} + { foo} + {plusfoo} + {middle} + {# some metacomment} + {# another metacomment} + {end} + {begin} + {minusfoo} + {middle} + {# some metacomment} + {end} "" + } \n +}] + +Verbatim guards can be used to force code line +interpretation of a block of lines even if some of them happen to look +like any other type of lines to docstrip. A verbatim guard has the +form '%<<[emph END-TAG]' and the verbatim block is terminated by the +first line that is exactly '%[emph END-TAG]'. +[example { + set text [join { + {begin} + {%<*myblock>} + {some stupid()} + { #computer<program>} + {%<<QQQ-98765} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + {%QQQ-98765} + { using*strange@programming<language>} + {%</myblock>} + {end} + } \n] + set res [docstrip::extract $text myblock -metaprefix {# }] + append res [docstrip::extract $text {}] +}] +sets $res to the result of +[example { + join { + {begin} + {some stupid()} + { #computer<program>} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + { using*strange@programming<language>} + {end} + {begin} + {end} "" + } \n +}] +The processing of verbatim guards takes place also inside blocks of +lines which due to some outer block guard will not be copied. +[para] + +The final piece of [syscmd docstrip] syntax is that extraction +stops at a line that is exactly "\endinput"; this is often used to +avoid copying random whitespace at the end of a file. In the unlikely +case that one wants such a code line, one can protect it with a +verbatim guard. + +[section Commands] + +The package defines two commands. + +[list_begin definitions] +[call [cmd docstrip::extract] [arg text] [arg terminals] [ + opt "[arg option] [arg value] ..." +]] + The [cmd extract] command docstrips the [arg text] and returns the + extracted lines of code, as a string with each line terminated with + a newline. The [arg terminals] is the list of those guard + expression terminals which should evaluate to true. + The available options are: + [list_begin options] + [opt_def -annotate [arg lines]] + Requests the specified number of lines of annotation to follow + each extracted line in the result. Defaults to 0. Annotation lines + are mostly useful when the extracted lines are to undergo some + further transformation. A first annotation line is a list of three + elements: line type, prefix removed in extraction, and prefix + inserted in extraction. The line type is one of: 'V' (verbatim), + 'M' (metacomment), '+' (+ or no modifier guard line), '-' (- + modifier guard line), '.' (normal line). A second annotation line + is the source line number. A third annotation line is the current + stack of block guards. Requesting more than three lines of + annotation is currently not supported. + [opt_def -metaprefix [arg string]] + The string by which the '%%' prefix of a metacomment line will + be replaced. Defaults to '%%'. For Tcl code this would typically + be '#'. + [opt_def -onerror [arg keyword]] + Controls what will be done when a format error in the [arg text] + being processed is detected. The settings are: + [list_begin definitions] + [def [const ignore]] + Just ignore the error; continue as if nothing happened. + [def [const puts]] + Write an error message to [const stderr], then continue + processing. + [def [const throw]] + Throw an error. The [option -errorcode] is set to a list whose + first element is [const DOCSTRIP], second element is the + type of error, and third element is the line number where + the error is detected. This is the default. + [list_end] + [opt_def -trimlines [arg boolean]] + Controls whether [emph spaces] at the end of a line should be + trimmed away before the line is processed. Defaults to true. + [list_end] + + It should be remarked that the [arg terminals] are often called + "options" in the context of the [syscmd docstrip] program, since + these specify which optional code fragments should be included. + +[call [cmd docstrip::sourcefrom] [arg filename] [arg terminals] [ + opt "[arg option] [arg value] ..." +]] + The [cmd sourcefrom] command is a docstripping emulation of + [cmd source]. It opens the file [arg filename], reads it, closes it, + docstrips the contents as specified by the [arg terminals], and + evaluates the result in the local context of the caller, during + which time the [cmd info] [method script] value will be the + [arg filename]. The options are passed on to [cmd fconfigure] to + configure the file before its contents are read. The + [option -metaprefix] is set to '#', all other [cmd extract] + options have their default values. +[list_end] + + +[section {Document structure}] + +The file format (as described above) determines whether a master +source code file can be processed correctly by [syscmd docstrip], +but the usefulness of the format is to no little part also dependent +on that the code and comment lines together constitute a well-formed +document. +[para] + +For a document format that does not require any non-Tcl software, see +the [cmd ddt2man] command in the [package docstrip::util] package. It +is suggested that files employing that document format are given the +suffix [file .ddt], to distinguish them from the more traditional +LaTeX-based [file .dtx] files. +[para] + +Master source files with [file .dtx] extension are usually set up so +that they can be typeset directly by [syscmd latex] without any +support from other files. This is achieved by beginning the file +with the lines +[example_begin] + % \iffalse + %<*driver> + \documentclass{tclldoc} + \begin{document} + \DocInput{[emph filename.dtx]} + \end{document} + %</driver> + % \fi +[example_end] +or some variation thereof. The trick is that the file gets read twice. +With normal LaTeX reading rules, the first two lines are comments and +therefore ignored. The third line is the document preamble, the fourth +line begins the document body, and the sixth line ends the document, +so LaTeX stops there [vset emdash] non-comments below that point in +the file are never subjected to the normal LaTeX reading rules. Before +that, however, the \DocInput command on the fifth line is processed, +and that does two things: it changes the interpretation of '%' from +"comment" to "ignored", and it inputs the file specified in the +argument (which is normally the name of the file the command is in). +It is this second time that the file is being read that the comments +and code in it are typeset. +[para] + +The function of the \iffalse ... \fi is to skip lines two to seven +on this second time through; this is similar to the "if 0 { ... }" +idiom for block comments in Tcl code, and it is needed here because +(amongst other things) the \documentclass command may only be +executed once. The function of the <driver> guards is to prevent this +short piece of LaTeX code from being extracted by [syscmd docstrip]. +The total effect is that the file can function both as a LaTeX +document and as a [syscmd docstrip] master source code file. +[para] + +It is not necessary to use the tclldoc document class, but that does +provide a number of features that are convenient for [file .dtx] +files containing Tcl code. More information on this matter can be +found in the references above. + +[manpage_end] diff --git a/tcllib/modules/docstrip/docstrip.tcl b/tcllib/modules/docstrip/docstrip.tcl new file mode 100644 index 0000000..fa6399b --- /dev/null +++ b/tcllib/modules/docstrip/docstrip.tcl @@ -0,0 +1,163 @@ +## +## This is the file `docstrip.tcl', +## generated with the SAK utility +## (sak docstrip/regen). +## +## The original source files were: +## +## tcldocstrip.dtx (with options: `pkg') +## +## In other words: +## ************************************** +## * This Source is not the True Source * +## ************************************** +## the true source is the file from which this one was generated. +## +package require Tcl 8.4 +package provide docstrip 1.2 +namespace eval docstrip { + namespace export extract sourcefrom +} +proc docstrip::extract {text terminals args} { + array set O { + -annotate 0 + -metaprefix %% + -onerror throw + -trimlines 1 + } + array set O $args + foreach t $terminals {set T($t) ""} + set stripped "" + set block_stack [list] + set offlevel 0 + set verbatim 0 + set lineno 0 + foreach line [split $text \n] { + incr lineno + if {$O(-trimlines)} then { + set line [string trimright $line " "] + } + if {$verbatim} then { + if {$line eq $endverbline} then { + set verbatim 0 + continue + } elseif {$offlevel} then { + continue + } + append stripped $line \n + if {$O(-annotate)>=1} then {append stripped {V "" ""} \n} + } else { + switch -glob -- $line %%* { + if {!$offlevel} then { + append stripped $O(-metaprefix)\ + [string range $line 2 end] \n + if {$O(-annotate)>=1} then { + append stripped [list M %% $O(-metaprefix)] \n + } + } + } %<<* { + set endverbline "%[string range $line 3 end]" + set verbatim 1 + continue + } %<* { + if {![ + regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\ + modifier expression line + ]} then { + extract,error BADGUARD\ + "Malformed guard \"\n$line\n\"" + "Malformed guard on line $lineno" + continue + } + regsub -all -- {\\|\{|\}|\$|\[|\]| |;} $expression\ + {\\&} E + regsub -all -- {,} $E {|} E + regsub -all -- {[^()|&!]+} $E {[info exists T(&)]} E + if {[catch {expr $E} val]} then { + extract,error EXPRERR\ + "Error in expression <$expression> ignored"\ + "docstrip: $val" + set val -1 + } + switch -exact -- $modifier * { + lappend block_stack $expression + if {$offlevel || !$val} then {incr offlevel} + continue + } / { + if {![llength $block_stack]} then { + extract,error SPURIOUS\ + "Spurious end block </$expression> ignored"\ + "Spurious end block </$expression>" + } else { + if {[string compare $expression\ + [lindex $block_stack end]]} then { + extract,error MISMATCH\ + "Found </$expression> instead of\ + </[lindex $block_stack end]>" + } + if {$offlevel} then {incr offlevel -1} + set block_stack [lreplace $block_stack end end] + } + continue + } - { + if {$offlevel || $val} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then { + append stripped [list - %<-${expression}> ""] \n + } + } default { + if {$offlevel || !$val} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then { + append stripped\ + [list + %<${modifier}${expression}> ""] \n + } + } + } %* {continue}\ + {\\endinput} { + break + } default { + if {$offlevel} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then {append stripped {. "" ""} \n} + } + } + if {$O(-annotate)>=2} then {append stripped $lineno \n} + if {$O(-annotate)>=3} then {append stripped $block_stack \n} + } + return $stripped +} +proc docstrip::extract,error {situation message {errmessage ""}} { + upvar 1 O(-onerror) onerror lineno lineno + switch -- [string tolower $onerror] "puts" { + puts stderr "docstrip: $message on line $lineno." + } "ignore" {} default { + if {$errmessage ne ""} then { + error $errmessage "" [list DOCSTRIP $situation $lineno] + } else { + error $message "" [list DOCSTRIP $situation $lineno] + } + } +} +proc docstrip::sourcefrom {name terminals args} { + set F [open $name r] + if {[llength $args]} then { + eval [linsert $args 0 fconfigure $F] + } + set text [read $F] + close $F + set oldscr [info script] + info script $name + set code [catch { + uplevel 1 [extract $text $terminals -metaprefix #] + } res] + info script $oldscr + if {$code == 1} then { + error $res $::errorInfo $::errorCode + } else { + return $res + } +} +## +## +## End of file `docstrip.tcl'.
\ No newline at end of file diff --git a/tcllib/modules/docstrip/docstrip.test b/tcllib/modules/docstrip/docstrip.test new file mode 100644 index 0000000..f7ffd94 --- /dev/null +++ b/tcllib/modules/docstrip/docstrip.test @@ -0,0 +1,243 @@ +## +## This is the file `docstrip.test', +## generated with the SAK utility +## (sak docstrip/regen). +## +## The original source files were: +## +## tcldocstrip.dtx (with options: `test tcllibtest') +## +## In other words: +## ************************************** +## * This Source is not the True Source * +## ************************************** +## the true source is the file from which this one was generated. +## +source [file join\ + [file dirname [file dirname [file join [pwd] [info script]]]]\ + devtools testutilities.tcl] +testsNeedTcl 8.4 +testsNeedTcltest 2 +testing {useLocal docstrip.tcl docstrip} +variable docstrip_sources_dir [localPath {}] +tcltest::testConstraint docstripSourcesAvailable [expr {[ + file exists [file join $docstrip_sources_dir docstrip.tcl] +] && [ + file exists [file join $docstrip_sources_dir tcldocstrip.dtx] +]}] +tcltest::test docstrip-1.1 {code/comment line distinction} -body { + docstrip::extract [join { + {% comment} + {% more comment !"#$%&/(} + {some command} + { % blah $blah "Not a comment."} + {% abc; this is comment} + {# def; this is code} + {ghi} + {% jkl} + } \n] {} +} -result [ + join { + {some command} + { % blah $blah "Not a comment."} + {# def; this is code} + {ghi} "" + } \n +] +tcltest::test docstrip-1.2 {blocks and nesting} -body { + set text [join { + {begin} + {%<*foo>} + {1} + {%<*bar>} + {2} + {%</bar>} + {%<*!bar>} + {3} + {%</!bar>} + {4} + {%</foo>} + {5} + {%<*bar>} + {6} + {%</bar>} + {end} + } \n] + set res [docstrip::extract $text foo] + append res [docstrip::extract $text {foo bar}] + append res [docstrip::extract $text bar] +} -result [ + join { + {begin} + {1} + {3} + {4} + {5} + {end} + {begin} + {1} + {2} + {4} + {5} + {6} + {end} + {begin} + {5} + {6} + {end} "" + } \n +] +tcltest::test docstrip-1.3 {plusminus guards and metacomments} -body { + set text [join { + {begin} + {%<foo> foo} + {%<+foo>plusfoo} + {%<-foo>minusfoo} + {middle} + {%% some metacomment} + {%<*foo>} + {%%another metacomment} + {%</foo>} + {end} + } \n] + set res [docstrip::extract $text foo -metaprefix {# }] + append res [docstrip::extract $text bar -metaprefix {#}] +} -result [ + join { + {begin} + { foo} + {plusfoo} + {middle} + {# some metacomment} + {# another metacomment} + {end} + {begin} + {minusfoo} + {middle} + {# some metacomment} + {end} "" + } \n +] +tcltest::test docstrip-1.4 {verbatim mode} -body { + set text [join { + {begin} + {%<*myblock>} + {some stupid()} + { #computer<program>} + {%<<QQQ-98765} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + {%QQQ-98765} + { using*strange@programming<language>} + {%</myblock>} + {end} + } \n] + set res [docstrip::extract $text myblock -metaprefix {# }] + append res [docstrip::extract $text {}] +} -result [ + join { + {begin} + {some stupid()} + { #computer<program>} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + { using*strange@programming<language>} + {end} + {begin} + {end} "" + } \n +] +tcltest::test docstrip-1.5 {annotation} -body { + set text [join { + {begin} + {%<*myblock>} + {some stupid()} + {%<foo> #computer<program>} + {%<<QQQ-98765} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + {%QQQ-98765} + { using*strange@programming<language>} + {%</myblock>} + {%%end} + } \n] + docstrip::extract $text {myblock foo} -metaprefix {# } -annotate 3 +} -result [ + join { + {begin} {. "" ""} 1 {} + {some stupid()} {. "" ""} 3 myblock + { #computer<program>} {+ %<foo> {}} 4 myblock + {% These three lines are copied verbatim (including percents} + {V "" ""} 6 myblock + {%% even if -metaprefix is something different than %%).} + {V "" ""} 7 myblock + {%</myblock>} {V "" ""} 8 myblock + { using*strange@programming<language>} {. "" ""} 10 myblock + {# end} {M %% {# }} 12 {} + "" + } \n +] +tcltest::test docstrip-2.1 {have docstrip extract itself} -constraints { + docstripSourcesAvailable +} -body { + # First read in the ready-stripped file, but gobble the preamble and + # postamble, as those are a bit messy to reproduce. + set F [open [file join $docstrip_sources_dir docstrip.tcl] r] + regsub -all -- {(^|\n)#[^\n]*} [read $F] {} stripped + close $F + # Then read the master source and strip it manually. + set F [open [file join $docstrip_sources_dir tcldocstrip.dtx] r] + set source [read $F] + close $F + set stripped2 [docstrip::extract $source pkg -metaprefix ##] + # Finally compare the two. + if {[string trim $stripped \n] ne [string trim $stripped2 \n]} then { + error "$strippped\n ne \n$stripped2" + } +} +tcltest::test docstrip-2.2 {soucefrom} -setup { + set dtxname [tcltest::makeFile [join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + {set baz 1} + {%</foo>} + {%<-foo>return} + {%</bar>} + {puts $baz} + {puts [file tail [info script]]} + {%<*!foo>} + {puts C} + "%% Tricky comment; guess what comes next\\" + {%</!foo>} + {incr baz} + {puts "baz=$baz"} + } \n] te27st01.dtx] +} -body { + set baz 0 + puts [info script] + docstrip::sourcefrom $dtxname {foo bar} + puts [info script] + docstrip::sourcefrom $dtxname {} + docstrip::sourcefrom $dtxname {bar} + puts $baz +} -cleanup { + tcltest::removeFile $dtxname +} -output [join [list\ + [info script]\ + {A} {B} {1} {1} {te27st01.dtx} {baz=2}\ + [info script]\ + {A} {2} {te27st01.dtx} {C} {baz=2}\ + {A} {B}\ + {2} "" +] \n] +testsuiteCleanup +## +## +## End of file `docstrip.test'.
\ No newline at end of file diff --git a/tcllib/modules/docstrip/docstrip_util.man b/tcllib/modules/docstrip/docstrip_util.man new file mode 100644 index 0000000..35e73c7 --- /dev/null +++ b/tcllib/modules/docstrip/docstrip_util.man @@ -0,0 +1,586 @@ +[vset VERSION 1.3.1] +[manpage_begin docstrip_util n [vset VERSION]] +[see_also docstrip] +[see_also doctools] +[see_also doctools_fmt] +[keywords .ddt] +[keywords .dtx] +[keywords catalogue] +[keywords diff] +[keywords docstrip] +[keywords doctools] +[keywords documentation] +[keywords LaTeX] +[keywords {literate programming}] +[keywords module] +[keywords {package indexing}] +[keywords patch] +[keywords source] +[keywords {Tcl module}] +[copyright "2003\u20132010 Lars Hellstr\u00F6m\ + <Lars dot Hellstrom at residenset dot net>"] +[moddesc {Literate programming tool}] +[titledesc {Docstrip-related utilities}] +[category {Documentation tools}] +[require Tcl 8.4] +[require docstrip [opt 1.2]] +[require docstrip::util [opt [vset VERSION]]] +[vset emdash \u2014] +[description] +The [package docstrip::util] package is meant for collecting various +utility procedures that are mainly useful at installation or +development time. It is separate from the base package to avoid +overhead when the latter is used to [cmd source] code. +[para] +[section {Package indexing commands}] + +Like raw [file .tcl] files, code lines in docstrip source files can +be searched for package declarations and corresponding indices +constructed. A complication is however that one cannot tell from the +code blocks themselves which will fit together to make a working +package; normally that information would be found in an accompanying +[file .ins] file, but parsing one of those is not an easy task. +Therefore [package docstrip::util] introduces an alternative encoding +of such information, in the form of a declarative Tcl script: the +[term catalogue] (of the contents in a source file). +[para] + +The special commands which are available inside a catalogue are: +[list_begin definitions] +[call [cmd pkgProvide] [arg name] [arg version] [arg terminals]] + Declares that the code for a package with name [arg name] and + version [arg version] is made up from those modules in the source + file which are selected by the [arg terminals] list of guard + expression terminals. This code should preferably not contain a + [cmd {package}] [method {provide}] command for the package, as one + will be provided by the package loading mechanisms. +[call [cmd pkgIndex] [opt "[arg terminal] ..."]] + Declares that the code for a package is made up from those modules + in the source file which are selected by the listed guard + expression [arg terminal]s. The name and version of this package is + determined from [cmd {package}] [method {provide}] command(s) found + in that code (hence there must be such a command in there). +[call [cmd fileoptions] [opt "[arg option] [arg value] ..."]] + Declares the [cmd fconfigure] options that should be in force when + reading the source; this can usually be ignored for pure ASCII + files, but if the file needs to be interpreted according to some + other [option -encoding] then this is how to specify it. The + command should normally appear first in the catalogue, as it takes + effect only for commands following it. +[list_end] +Other Tcl commands are supported too [vset emdash] a catalogue is +parsed by being evaluated in a safe interpreter [vset emdash] but they +are rarely needed. To allow for future extensions, unknown commands +in the catalogue are silently ignored. +[para] + +To simplify distribution of catalogues together with their source +files, the catalogue is stored [emph {in the source file itself}] as +a module selected by the terminal '[const docstrip.tcl::catalogue]'. +This supports both the style of collecting all catalogue lines in one +place and the style of putting each catalogue line in close proximity +of the code that it declares. +[para] + +Putting catalogue entries next to the code they declare may look as +follows +[example { +% First there's the catalogue entry +% \begin{tcl} +%<docstrip.tcl::catalogue>pkgProvide foo::bar 1.0 {foobar load} +% \end{tcl} +% second a metacomment used to include a copyright message +% \begin{macrocode} +%<*foobar> +%% This file is placed in the public domain. +% \end{macrocode} +% third the package implementation +% \begin{tcl} +namespace eval foo::bar { + # ... some clever piece of Tcl code elided ... +% \end{tcl} +% which at some point may have variant code to make use of a +% |load|able extension +% \begin{tcl} +%<*load> + load [file rootname [info script]][info sharedlibextension] +%</load> +%<*!load> + # ... even more clever scripted counterpart of the extension + # also elided ... +%</!load> +} +%</foobar> +% \end{tcl} +% and that's it! +}] +The corresponding set-up with [cmd pkgIndex] would be +[example { +% First there's the catalogue entry +% \begin{tcl} +%<docstrip.tcl::catalogue>pkgIndex foobar load +% \end{tcl} +% second a metacomment used to include a copyright message +% \begin{tcl} +%<*foobar> +%% This file is placed in the public domain. +% \end{tcl} +% third the package implementation +% \begin{tcl} +package provide foo::bar 1.0 +namespace eval foo::bar { + # ... some clever piece of Tcl code elided ... +% \end{tcl} +% which at some point may have variant code to make use of a +% |load|able extension +% \begin{tcl} +%<*load> + load [file rootname [info script]][info sharedlibextension] +%</load> +%<*!load> + # ... even more clever scripted counterpart of the extension + # also elided ... +%</!load> +} +%</foobar> +% \end{tcl} +% and that's it! +}] +[list_begin definitions] +[call [cmd docstrip::util::index_from_catalogue] [arg dir]\ + [arg pattern] [opt "[arg option] [arg value] ..."]] + This command is a sibling of the standard [cmd pkg_mkIndex] + command, in that it adds package entries to [file pkgIndex.tcl] + files. The difference is that it indexes [syscmd docstrip]-style + source files rather than raw [file .tcl] or loadable library files. + Only packages listed in the catalogue of a file are considered. + [para] + + The [arg dir] argument is the directory in which to look for files + (and whose [file pkgIndex.tcl] file should be amended). + The [arg pattern] argument is a [cmd glob] pattern of files to look + into; a typical value would be [const *.dtx] or + [const *.{dtx,ddt}]. Remaining arguments are option-value pairs, + where the supported options are: + [list_begin options] + [opt_def -recursein [arg dirpattern]] + If this option is given, then the [cmd index_from_catalogue] + operation will be repeated in each subdirectory whose name + matches the [arg dirpattern]. [option -recursein] [const *] will + cause the entire subtree rooted at [arg dir] to be indexed. + [opt_def -sourceconf [arg dictionary]] + Specify [cmd fileoptions] to use when reading the catalogues of + files (and also for reading the packages if the catalogue does + not contain a [cmd fileoptions] command). Defaults to being + empty. Primarily useful if your system encoding is very different + from that of the source file (e.g., one is a two-byte encoding + and the other is a one-byte encoding). [const ascii] and + [const utf-8] are not very different in that sense. + [opt_def -options [arg terminals]] + The [arg terminals] is a list of terminals in addition to + [const docstrip.tcl::catalogue] that should be held as true when + extracting the catalogue. Defaults to being empty. This makes it + possible to make use of "variant sections" in the catalogue + itself, e.g. gaurd some entries with an extra "experimental" and + thus prevent them from appearing in the index unless that is + generated with "experimental" among the [option -options]. + [opt_def -report [arg boolean]] + If the [arg boolean] is true then the return value will be a + textual, probably multiline, report on what was done. Defaults + to false, in which case there is no particular return value. + [opt_def -reportcmd [arg commandPrefix]] + Every item in the report is handed as an extra argument to the + command prefix. Since [cmd index_from_catalogue] would typically + be used at a rather high level in installation scripts and the + like, the [arg commandPrefix] defaults to + "[cmd puts] [const stdout]". + Use [cmd list] to effectively disable this feature. The return + values from the prefix are ignored. + [list_end] + + The [cmd {package ifneeded}] scripts that are generated contain + one [cmd {package require docstrip}] command and one + [cmd docstrip::sourcefrom] command. If the catalogue entry was + of the [cmd pkgProvide] kind then the [cmd {package ifneeded}] + script also contains the [cmd {package provide}] command. + [para] + + Note that [cmd index_from_catalogue] never removes anything from an + existing [file pkgIndex.tcl] file. Hence you may need to delete it + (or have [cmd pkg_mkIndex] recreate it from scratch) before running + [cmd index_from_catalogue] to update some piece of information, such + as a package version number. + [para] +[call [cmd docstrip::util::modules_from_catalogue] [arg target]\ + [arg source] [opt "[arg option] [arg value] ..."]] + This command is an alternative to [cmd index_from_catalogue] which + creates Tcl Module ([file .tm]) files rather than + [file pkgIndex.tcl] entries. Since this action is more similar to + what [syscmd docstrip] classically does, it has features for + putting pre- and postambles on the generated files. + [para] + + The [arg source] argument is the name of the source file to + generate [file .tm] files from. The [arg target] argument is the + directory which should count as a module path, i.e., this is what + the relative paths derived from package names are joined to. The + supported options are: + [list_begin options] + [opt_def -preamble [arg message]] + A message to put in the preamble (initial block of comments) of + generated files. Defaults to a space. May be several lines, which + are then separated by newlines. Traditionally used for copyright + notices or the like, but metacomment lines provide an alternative + to that. + [opt_def -postamble [arg message]] + Like [option -preamble], but the message is put at the end of the + file instead of the beginning. Defaults to being empty. + [opt_def -sourceconf [arg dictionary]] + Specify [cmd fileoptions] to use when reading the catalogue of + the [arg source] (and also for reading the packages if the + catalogue does not contain a [cmd fileoptions] command). Defaults + to being empty. Primarily useful if your system encoding is very + different from that of the source file (e.g., one is a two-byte + encoding and the other is a one-byte encoding). [const ascii] and + [const utf-8] are not very different in that sense. + [opt_def -options [arg terminals]] + The [arg terminals] is a list of terminals in addition to + [const docstrip.tcl::catalogue] that should be held as true when + extracting the catalogue. Defaults to being empty. This makes it + possible to make use of "variant sections" in the catalogue + itself, e.g. gaurd some entries with an extra "experimental" guard + and thus prevent them from contributing packages unless those are + generated with "experimental" among the [option -options]. + [opt_def -formatpreamble [arg commandPrefix]] + Command prefix used to actually format the preamble. Takes four + additional arguments [arg message], [arg targetFilename], + [arg sourceFilename], and [arg terminalList] and returns a fully + formatted preamble. Defaults to using [cmd classical_preamble] + with a [arg metaprefix] of '##'. + [opt_def -formatpostamble [arg commandPrefix]] + Command prefix used to actually format the postamble. Takes four + additional arguments [arg message], [arg targetFilename], + [arg sourceFilename], and [arg terminalList] and returns a fully + formatted postamble. Defaults to using [cmd classical_postamble] + with a [arg metaprefix] of '##'. + [opt_def -report [arg boolean]] + If the [arg boolean] is true (which is the default) then the return + value will be a textual, probably multiline, report on what was + done. If it is false then there is no particular return value. + [opt_def -reportcmd [arg commandPrefix]] + Every item in the report is handed as an extra argument to this + command prefix. Defaults to [cmd list], which effectively disables + this feature. The return values from the prefix are ignored. Use + for example "[cmd puts] [const stdout]" to get report items + written immediately to the terminal. + [list_end] + An existing file of the same name as one to be created will be + overwritten. +[call [cmd docstrip::util::classical_preamble] [arg metaprefix]\ + [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]] + This command returns a preamble in the classical + [syscmd docstrip] style +[example { +## +## This is `TARGET', +## generated by the docstrip::util package. +## +## The original source files were: +## +## SOURCE (with options: `foo,bar') +## +## Some message line 1 +## line2 +## line3 +}] + if called as +[example_begin] +docstrip::util::classical_preamble {##}\ + "\nSome message line 1\nline2\nline3" TARGET SOURCE {foo bar} +[example_end] + The command supports preambles for files generated from multiple + sources, even though [cmd modules_from_catalogue] at present does + not need that. +[call [cmd docstrip::util::classical_postamble] [arg metaprefix]\ + [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]] + This command returns a postamble in the classical + [syscmd docstrip] style +[example { +## Some message line 1 +## line2 +## line3 +## +## End of file `TARGET'. +}] + if called as +[example_begin] +docstrip::util::classical_postamble {##}\ + "Some message line 1\nline2\nline3" TARGET SOURCE {foo bar} +[example_end] + In other words, the [arg source] and [arg terminals] arguments are + ignored, but supported for symmetry with [cmd classical_preamble]. +[call [cmd docstrip::util::packages_provided] [arg text]\ + [opt [arg setup-script]]] + This command returns a list where every even index element is the + name of a package [cmd provide]d by [arg text] when that is + evaluated as a Tcl script, and the following odd index element is + the corresponding version. It is used to do package indexing of + extracted pieces of code, in the manner of [cmd pkg_mkIndex]. + [para] + + One difference to [cmd pkg_mkIndex] is that the [arg text] gets + evaluated in a safe interpreter. [cmd {package require}] commands + are silently ignored, as are unknown commands (which includes + [cmd source] and [cmd load]). Other errors cause + processing of the [arg text] to stop, in which case only those + package declarations that had been encountered before the error + will be included in the return value. + [para] + + The [arg setup-script] argument can be used to customise the + evaluation environment, if the code in [arg text] has some very + special needs. The [arg setup-script] is evaluated in the local + context of the [cmd packages_provided] procedure just before the + [arg text] is processed. At that time, the name of the slave + command for the safe interpreter that will do this processing is + kept in the local variable [var c]. To for example copy the + contents of the [var ::env] array to the safe interpreter, one + might use a [arg setup-script] of + [example { $c eval [list array set env [array get ::env]]}] +[list_end] + +[section {Source processing commands}] + +Unlike the previous group of commands, which would use +[cmd docstrip::extract] to extract some code lines and then process +those further, the following commands operate on text consisting of +all types of lines. + +[list_begin definitions] +[call [cmd docstrip::util::ddt2man] [arg text]] + The [cmd ddt2man] command reformats [arg text] from the general + [syscmd docstrip] format to [package doctools] [file .man] format + (Tcl Markup Language for Manpages). The different line types are + treated as follows: + [list_begin definitions] + [def {comment and metacomment lines}] + The '%' and '%%' prefixes are removed, the rest of the text is + kept as it is. + [def {empty lines}] + These are kept as they are. (Effectively this means that they will + count as comment lines after a comment line and as code lines + after a code line.) + [def {code lines}] + [cmd example_begin] and [cmd example_end] commands are placed + at the beginning and end of every block of consecutive code + lines. Brackets in a code line are converted to [cmd lb] and + [cmd rb] commands. + [def {verbatim guards}] + These are processed as usual, so they do not show up in the + result but every line in a verbatim block is treated as a code + line. + [def {other guards}] + These are treated as code lines, except that the actual guard is + [cmd emph]asised. + [list_end] + + At the time of writing, no project has employed [package doctools] + markup in master source files, so experience of what works well is + not available. A source file could however look as follows +[example { +% [manpage_begin gcd n 1.0] +% [keywords divisor] +% [keywords math] +% [moddesc {Greatest Common Divisor}] +% [require gcd [opt 1.0]] +% [description] +% +% [list_begin definitions] +% [call [cmd gcd] [arg a] [arg b]] +% The [cmd gcd] procedure takes two arguments [arg a] and [arg b] which +% must be integers and returns their greatest common divisor. +proc gcd {a b} { +% The first step is to take the absolute values of the arguments. +% This relieves us of having to worry about how signs will be treated +% by the remainder operation. + set a [expr {abs($a)}] + set b [expr {abs($b)}] +% The next line does all of Euclid's algorithm! We can make do +% without a temporary variable, since $a is substituted before the +% [lb]set a $b[rb] and thus continues to hold a reference to the +% "old" value of [var a]. + while {$b>0} { set b [expr { $a % [set a $b] }] } +% In Tcl 8.3 we might want to use [cmd set] instead of [cmd return] +% to get the slight advantage of byte-compilation. +%<tcl83> set a +%<!tcl83> return $a +} +% [list_end] +% +% [manpage_end] +}] + If the above text is fed through [cmd docstrip::util::ddt2man] then + the result will be a syntactically correct [package doctools] + manpage, even though its purpose is a bit different. + [para] + + It is suggested that master source code files with [package doctools] + markup are given the suffix [file .ddt], hence the "ddt" in + [cmd ddt2man]. + +[call [cmd docstrip::util::guards] [arg subcmd] [arg text]] + The [cmd guards] command returns information (mostly of a + statistical nature) about the ordinary docstrip guards that occur + in the [arg text]. The [arg subcmd] selects what is returned. + + [list_begin definitions] + [def [method counts]] + List the guard expression terminals with counts. The format of + the return value is a dictionary which maps the terminal name to + the number of occurencies of it in the file. + [def [method exprcount]] + List the guard expressions with counts. The format of the return + value is a dictionary which maps the expression to the number of + occurencies of it in the file. + [def [method exprerr]] + List the syntactically incorrect guard expressions (e.g. + parentheses do not match, or a terminal is missing). The return + value is a list, with the elements in no particular order. + [def [method expressions]] + List the guard expressions. The return value is a list, with the + elements in no particular order. + [def [method exprmods]] + List the guard expressions with modifiers. The format of the return + value is a dictionary where each index is a guard expression and + each entry is a string with one character for every guard line that + has this expression. The characters in the entry specify what + modifier was used in that line: +, -, *, /, or (for guard without + modifier:) space. This is the most primitive form of the + information gathered by [cmd guards]. + [def [method names]] + List the guard expression terminals. The return value is a list, + with the elements in no particular order. + [def [method rotten]] + List the malformed guard lines (this does not include lines where + only the expression is malformed, though). The format of the return + value is a dictionary which maps line numbers to their contents. + [list_end] +[call [cmd docstrip::util::patch] [arg source-var] [arg terminals]\ + [arg fromtext] [arg diff] [opt "[arg option] [arg value] ..."]] + This command tries to apply a [syscmd diff] file (for example a + contributed patch) that was computed for a generated file to the + [syscmd docstrip] source. This can be useful if someone has + edited a generated file, thus mistaking it for being the source. + This command makes no presumptions which are specific for the case + that the generated file is a Tcl script. + [para] + + [cmd patch] requires that the source file to patch is kept as a + list of lines in a variable, and the name of that variable in the + calling context is what goes into the [arg source-var] argument. + The [arg terminals] is the list of terminals used to extract the + file that has been patched. The [arg diff] is the actual diff to + apply (in a format as explained below) and the [arg fromtext] is + the contents of the file which served as "from" when the diff was + computed. Options can be used to further control the process. + [para] + + The process works by "lifting" the hunks in the [arg diff] from + generated to source file, and then applying them to the elements of + the [arg source-var]. In order to do this lifting, it is necessary + to determine how lines in the [arg fromtext] correspond to elements + of the [arg source-var], and that is where the [arg terminals] come + in; the source is first [cmd extract]ed under the given + [arg terminals], and the result of that is then matched against + the [arg fromtext]. This produces a map which translates line + numbers stated in the [arg diff] to element numbers in + [arg source-var], which is what is needed to lift the hunks. + [para] + + The reason that both the [arg terminals] and the [arg fromtext] + must be given is twofold. First, it is very difficult to keep track + of how many lines of preamble are supplied some other way than by + copying lines from source files. Second, a generated file might + contain material from several source files. Both make it impossible + to predict what line number an extracted file would have in the + generated file, so instead the algorithm for computing the line + number map looks for a block of lines in the [arg fromtext] which + matches what can be extracted from the source. This matching is + affected by the following options: + [list_begin options] + [opt_def -matching [arg mode]] + How equal must two lines be in order to match? The supported + [arg mode]s are: + [list_begin definitions] + [def [const exact]] + Lines must be equal as strings. This is the default. + [def [const anyspace]] + All sequences of whitespace characters are converted to single + spaces before comparing. + [def [const nonspace]] + Only non-whitespace characters are considered when comparing. + [def [const none]] + Any two lines are considered to be equal. + [list_end] + [opt_def -metaprefix [arg string]] + The [option -metaprefix] value to use when extracting. Defaults + to "%%", but for Tcl code it is more likely that "#" or "##" had + been used for the generated file. + [opt_def -trimlines [arg boolean]] + The [option -trimlines] value to use when extracting. Defaults to + true. + [list_end] + + The return value is in the form of a unified diff, containing only + those hunks which were not applied or were only partially applied; + a comment in the header of each hunk specifies which case is at + hand. It is normally necessary to manually review both the return + value from [cmd patch] and the patched text itself, as this command + cannot adjust comment lines to match new content. + [para] + + An example use would look like +[example_begin] +set sourceL [lb]split [lb]docstrip::util::thefile from.dtx[rb] \n[rb] +set terminals {foo bar baz} +set fromtext [lb]docstrip::util::thefile from.tcl[rb] +set difftext [lb]exec diff --unified from.tcl to.tcl[rb] +set leftover [lb]docstrip::util::patch sourceL $terminals $fromtext\ + [lb]docstrip::util::import_unidiff $difftext[rb] -metaprefix {#}[rb] +set F [lb]open to.dtx w[rb]; puts $F [lb]join $sourceL \n[rb]; close $F +return $leftover +[example_end] + Here, [file from.dtx] was used as source for [file from.tcl], which + someone modified into [file to.tcl]. We're trying to construct a + [file to.dtx] which can be used as source for [file to.tcl]. +[call [cmd docstrip::util::thefile] [arg filename] [ + opt "[arg option] [arg value] ..." +]] + The [cmd thefile] command opens the file [arg filename], reads it to + end, closes it, and returns the contents (dropping a final newline + if there is one). The option-value pairs are + passed on to [cmd fconfigure] to configure the open file channel + before anything is read from it. +[call [cmd docstrip::util::import_unidiff] [arg diff-text]\ + [opt [arg warning-var]]] + This command parses a unified ([syscmd diff] flags [option -U] and + [option --unified]) format diff into the list-of-hunks format + expected by [cmd docstrip::util::patch]. The [arg diff-text] + argument is the text to parse and the [arg warning-var] is, if + specified, the name in the calling context of a variable to which + any warnings about parsing problems will be [cmd append]ed. + [para] + + The return value is a list of [term hunks]. Each hunk is a list of + five elements "[arg start1] [arg end1] [arg start2] [arg end2] + [arg lines]". [arg start1] and [arg end1] are line numbers in the + "from" file of the first and last respectively lines of the hunk. + [arg start2] and [arg end2] are the corresponding line numbers in + the "to" file. Line numbers start at 1. The [arg lines] is a list + with two elements for each line in the hunk; the first specifies the + type of a line and the second is the actual line contents. The type + is [const -] for lines only in the "from" file, [const +] for lines + that are only in the "to" file, and [const 0] for lines that are + in both. +[list_end] +[manpage_end] diff --git a/tcllib/modules/docstrip/docstrip_util.tcl b/tcllib/modules/docstrip/docstrip_util.tcl new file mode 100644 index 0000000..b3a0009 --- /dev/null +++ b/tcllib/modules/docstrip/docstrip_util.tcl @@ -0,0 +1,649 @@ +## +## This is the file `docstrip_util.tcl', +## generated with the SAK utility +## (sak docstrip/regen). +## +## The original source files were: +## +## tcldocstrip.dtx (with options: `utilpkg') +## +## In other words: +## ************************************** +## * This Source is not the True Source * +## ************************************** +## the true source is the file from which this one was generated. +## +package require Tcl 8.4 +package require docstrip 1.2 +package provide docstrip::util 1.3.1 +namespace eval docstrip::util { + namespace export ddt2man guard patch thefile\ + packages_provided index_from_catalogue modules_from_catalogue\ + classical_preamble classical_postamble +} +namespace eval docstrip::util { + namespace import [namespace parent]::extract +} +proc docstrip::util::fileoptions {args} { + variable filename + variable thefile [eval [list thefile $filename] $args] + variable fileoptions $args +} +proc docstrip::util::Report {item} { + variable Report_store + if {$Report_store} then { + variable Report + lappend Report $item + } + variable Report_cmd + eval [linsert $Report_cmd end $item] +} +proc docstrip::util::index_from_catalogue {dir pattern args} { + array set O { + -options "" + -sourceconf "" + -report 0 + -reportcmd {puts stdout} + -RecursionDepth 0 + } + array set O $args + if {$O(-RecursionDepth)==0} then { + variable Report {} Report_store $O(-report) \ + Report_cmd $O(-reportcmd) + } + set targetFn [file join $dir pkgIndex.tcl] + Report "Entries will go to: $targetFn" + if {![file exists $targetFn]} then { + Report "Generating empty index file." + set F [open $targetFn w] + puts $F {# Tcl package index file, version 1.1} + puts $F {# This file is generated by the "pkg_mkIndex" command} + puts $F {# and sourced either when an application starts up or} + puts $F {# by a "package unknown" script. It invokes the} + puts $F {# "package ifneeded" command to set up package-related} + puts $F {# information so that packages will be loaded automatically} + puts $F {# in response to "package require" commands. When this} + puts $F {# script is sourced, the variable $dir must contain the} + puts $F {# full path name of this file's directory.} + close $F + } + set c [interp create -safe] + $c eval { + proc unknown args {} + } + $c alias pkgProvide [namespace which PkgProvide] + $c alias pkgIndex [namespace which PkgIndex] + $c alias fileoptions [namespace which fileoptions] + variable PkgIndex "" + foreach fn [glob -nocomplain -directory $dir -tails $pattern] { + Report "Processing file: $fn" + variable filename [file join $dir $fn] + variable fileoptions $O(-sourceconf) + variable thefile [eval [list thefile $filename] $fileoptions] + set catalogue [extract $thefile\ + [linsert $O(-options) 0 docstrip.tcl::catalogue]\ + -metaprefix {#} -onerror puts] + $c eval $catalogue + } + interp delete $c + if {$PkgIndex ne ""} then { + set F [open $targetFn {WRONLY APPEND}] + set cmd [list docstrip::util::index_from_catalogue $dir $pattern] + if {$O(-options) ne ""} then { + lappend cmd -options $O(-options) + } + if {$O(-sourceconf) ne ""} then { + lappend cmd -sourceconf $O(-sourceconf) + } + puts $F "\n## Appendix generated by:\n## $cmd$PkgIndex" + close $F + } + if {[info exists O(-recursein)]} then { + incr O(-RecursionDepth) + foreach fn [ + glob -nocomplain -tails -types d -directory $dir\ + $O(-recursein) + ] { + eval [list index_from_catalogue [file join $dir $fn] $pattern]\ + [array get O] + } + } + if {$O(-RecursionDepth)==0 && $O(-report)} then { + return [join $Report \n] + } +} +proc docstrip::util::PkgProvide {pkg ver terminals} { + if {[catch {package vcompare 0 $ver}]} then { + Report "Malformed version number $ver given for package $pkg." + return + } + variable PkgIndex + variable filename + variable fileoptions + append PkgIndex \n [list package ifneeded $pkg $ver] { "} + append PkgIndex [string map {\\ {\\} \$ {\$} \[ {\[} \" {\"}}\ + [list package provide $pkg $ver]] {; } + append PkgIndex {package require docstrip} {; } + append PkgIndex {[list docstrip::sourcefrom }\ + {[file join $dir } [list [file tail $filename]] {] }\ + [linsert $fileoptions 0 $terminals] {]"} +} +proc docstrip::util::PkgIndex {args} { + variable thefile + if {[catch { + packages_provided [extract $thefile $args -metaprefix {#}] + } res]} then { + if {[lindex $::errorCode 0] eq "DOCSTRIP"} then { + Report "Stripping error \"$res\"\nwhile indexing module\ + <[join $args ,]>." + } else { + Report "Code evaluation error:\n $res\nwhile indexing\ + module <[join $args ,]>." + } + } else { + variable filename + variable PkgIndex + variable fileoptions + foreach {pkg ver} $res { + append PkgIndex \n [list package ifneeded $pkg $ver] { "} + append PkgIndex {package require docstrip} {; } + append PkgIndex {[list docstrip::sourcefrom }\ + {[file join $dir } [list [file tail $filename]] {] }\ + [linsert $fileoptions 0 $args] {]"} + } + } +} +proc docstrip::util::modules_from_catalogue {target source args} { + array set Opt { + -formatpostamble {classical_postamble {##}} + -formatpreamble {classical_preamble {##}} + -options {} + -postamble {} + -preamble { } + -sourceconf {} + -report 1 + -reportcmd list + } + array set Opt $args + variable filename $source + variable fileoptions $Opt(-sourceconf) + variable thefile [eval [list thefile $source] $fileoptions] + variable Report {} Report_store $Opt(-report) \ + Report_cmd $Opt(-reportcmd) + set catalogue [extract $thefile\ + [linsert $Opt(-options) 0 docstrip.tcl::catalogue]\ + -metaprefix {#} -onerror puts] + set c [interp create -safe] + $c eval { + proc unknown args {} + } + $c alias pkgProvide\ + [namespace which GenerateNamedPkg] $target\ + [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\ + [linsert $Opt(-formatpostamble) end $Opt(-postamble)] + $c alias pkgIndex\ + [namespace which GeneratePkg] $target\ + [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\ + [linsert $Opt(-formatpostamble) end $Opt(-postamble)] + $c alias fileoptions [namespace which fileoptions] + $c eval $catalogue + interp delete $c + if {$Opt(-report)} then {return [join $Report \n]} +} +proc docstrip::util::GenerateNamedPkg\ + {target preamblecmd postamblecmd name version terminals} { + variable thefile + if {[catch { + extract $thefile $terminals -metaprefix {#} + } text]} then { + Report "Stripping error \"$text\"\nwhile indexing module\ + <[join $terminals ,]>." + } else { + variable filename + set module [format {%s-%s.tm}\ + [string trim [string map {:: /} $name] /] $version] + set modL [file split $module] + file mkdir [file join $target [file dirname $module]] + set F [open [file join $target $module] w] + fconfigure $F -encoding utf-8 + puts $F [eval $preamblecmd [list $module $filename $terminals]] + puts -nonewline $F $text + puts $F [eval $postamblecmd [list $module $filename $terminals]] + close $F + Report "Wrote $module" + } +} +proc docstrip::util::GeneratePkg {target preamblecmd postamblecmd args} { + variable thefile + if {[catch { + set text [extract $thefile $args -metaprefix {#}] + packages_provided $text + } res]} then { + if {[lindex $::errorCode 0] eq "DOCSTRIP"} then { + Report "Stripping error \"$res\"\nwhile indexing module\ + <[join $args ,]>." + } else { + Report "Code evaluation error:\n $res\nwhile indexing\ + module <[join $args ,]>." + } + } elseif {![llength $res]} then { + Report "Found no package in module <[join $args ,]>." + } else { + variable filename + set module [format {%s-%s.tm}\ + [string trim [string map {:: /} [lindex $res 0]] /]\ + [lindex $res 1]] + set modL [file split $module] + file mkdir [file join $target [file dirname $module]] + set F [open [file join $target $module] w] + fconfigure $F -encoding utf-8 + puts $F [eval $preamblecmd [list $module $filename $args]] + puts -nonewline $F $text + puts $F [eval $postamblecmd [list $module $filename $args]] + close $F + Report "Wrote $module" + foreach {pkg ver} [lreplace $res 0 1] { + set mod2 [format {%s-%s.tm}\ + [string trim [string map {:: /} $pkg] /] $ver] + set mod2L [file split $mod2] + file mkdir [file join $target [file dirname $mod2]] + set common 0 + foreach d1 $modL d2 $mod2L { + if {$d1 eq $d2} then {incr common} else {break} + } + set tail [lrange $modL $common end] + set script {[::info script]} + foreach d2 $mod2L { + if {[incr common -1] < 0} then { + set script "\[::file dirname $script\]" + } + } + set F [open [file join $target $mod2] w] + fconfigure $F -encoding utf-8 + puts $F "::source -encoding utf-8 \[::file join $script $tail\]" + close $F + Report "Wrote redirect $mod2" + } + } +} +proc docstrip::util::classical_preamble {metaprefix message target args} { + set res {""} + lappend res " This is `$target'," + lappend res { generated by the docstrip::util package.} + lappend res {} { The original source files were:} {} + foreach {source terminals} $args { + set line " [file tail $source]" + if {[llength $terminals]} then { + append line { (with options: `} [join $terminals ,] {')} + } + lappend res $line + } + foreach line [split $message \n] {lappend res " $line"} + return $metaprefix[join $res "\n$metaprefix"] +} +proc docstrip::util::classical_postamble {metaprefix message target args} { + set res {} + foreach line [split $message \n] {lappend res " $line"} + lappend res {} " End of file `$target'." + return $metaprefix[join $res "\n$metaprefix"] +} +proc docstrip::util::packages_provided {text {setup ""}} { + set c [interp create -safe] + $c eval { + proc tclPkgUnknown args {} + package unknown tclPkgUnknown + proc unknown {args} {} + proc auto_import {args} {} + } + $c hide package + $c alias package [namespace which packages_provided,package] $c + eval $setup + set package_list {} + catch {$c eval $text} + interp delete $c + return $package_list +} +proc docstrip::util::packages_provided,package {interp subcmd args} { + switch -- $subcmd { + r - re - req - requ - requi - requir - require { + return + } + pro - prov - provi - provid - provide { + if {[llength $args] == 2} then { + uplevel 1 [list lappend package_list] $args + } + } + } + eval [list $interp invokehidden package $subcmd] $args +} +proc docstrip::util::ddt2man {text} { + set wascode 0 + set verbatim 0 + set res "" + foreach line [split $text \n] { + if {$verbatim} then { + if {$line eq $endverbline} then { + set verbatim 0 + } else { + append res [string map {[ [lb] ] [rb]} $line] \n + } + } else { + switch -glob -- $line %%* { + if {$wacode} then { + append res {[example_end]} \n + set wascode 0 + } + append res [string range $line 2 end] \n + } %<<* { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + set endverbline "%[string range $line 3 end]" + set verbatim 1 + } %<* { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + set guard "" + regexp -- {(^%<[^>]*>)(.*)$} $line "" guard line + append res \[ [list emph $guard] \]\ + [string map {[ [lb] ] [rb]} $line] \n + } %* { + if {$wascode} then { + append res {[example_end]} \n + set wascode 0 + } + append res [string range $line 1 end] \n + } {\\endinput} { + break + } "" { + append res \n + } default { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + append res [string map {[ [lb] ] [rb]} $line] \n + } + } + } + if {$wascode} then {append res {[example_end]} \n} + return $res +} +proc docstrip::util::guards {subcmd text} { + set verbatim 0 + set lineno 1 + set badL {} + foreach line [split $text \n] { + if {$verbatim} then { + if {$line eq $endverbline} then {set verbatim 0} + } else { + switch -glob -- $line %<<* { + set endverbline "%[string range $line 3 end]" + set verbatim 1 + } %<* { + if {![ + regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\ + modifier expression line + ]} then { + lappend badL $lineno $line + } else { + if {$modifier eq ""} then {set modifier " "} + append E($expression) $modifier + } + } + } + incr lineno + } + if {$subcmd eq "rotten"} then {return $badL} + switch -- $subcmd "exprmods" { + return [array get E] + } "expressions" { + return [array names E] + } "exprerr" { + set res {} + foreach expr [array names E] { + regsub -all {[^()!,|&]+} $expr 0 e + regsub -all {,} $e {|} e + if {[catch {expr $e}]} then {lappend res $expr} + } + return $res + } + foreach name [array names E] { + set E($name) [string length $E($name)] + } + if {$subcmd eq "exprcounts"} then {return [array get E]} + foreach expr [array names E] { + foreach term [split $expr "()!,|&"] { + if {$term eq ""} then {continue} + if {![info exists T($term)]} then {set T($term) 0} + incr T($term) $E($expr) + } + } + switch -- $subcmd "counts" { + return [array get T] + } "names" { + return [array names T] + } default { + error "Unknown subcommand '$subcmd', must be one of:\ + counts, exprcounts, expressions, exprmods, names, rotten" + } +} +proc docstrip::util::patch {sourcevar termL fromtext diff args} { + upvar 1 $sourcevar SL + array set O {-trimlines 1 -matching exact} + array set O $args + set cmd [list extract [join $SL \n] $termL -annotate 2] + foreach opt {-metaprefix -trimlines} { + if {[info exists O($opt)]} then {lappend cmd $opt $O($opt)} + } + set EL [split [eval $cmd] \n] + lset EL end \n + set ptr 0 + set lineno 1 + set FL [list {}] + foreach line [split $fromtext \n] { + lappend FL $line + if {$O(-trimlines)} then {set line [string trimright $line " "]} + if {$line eq [lindex $EL $ptr]} then { + set lift($lineno) [lindex $EL [incr ptr]] + lset lift($lineno) 0 [expr { [lindex $EL [incr ptr]] - 1 }] + incr ptr + } + incr lineno + } + if {![array size lift]} then { + return -code error "The extract did not match any part of the\ + fromtext. Check the list of terminals and the options" + } + set RL [list] + set log [list] + foreach hunk [lsort -decreasing -integer -index 0 $diff] { + set replL [list] + set l1 [lindex $hunk 0] + set repl {0 -1} + set matches 1 + foreach {type line} [lindex $hunk 4] { + switch -glob -- $type {[0-]} { + switch -- $O(-matching) "exact" { + if {[lindex $FL $l1] ne $line} then {set matches 0} + } "nonspace" { + if {[regsub -all -- {\s} $line {}] ne\ + [regsub -all -- {\s} [lindex $FL $l1] {}]} then { + set matches 0 + } + } "anyspace" { + if {[regsub -all -- {\s+} $line { }] ne\ + [regsub -all -- {\s+} [lindex $FL $l1] { }]} then { + set matches 0 + } + } + } + switch -- $type synch { + if {[llength $repl]>2 ||\ + [lindex $repl 1]-[lindex $repl 0]>=0} then { + lappend replL $repl + } + set repl [list $l1 [expr {$l1-1}]] + } + { + lappend repl $line + } - { + lset repl 1 $l1 + incr l1 + } 0 { + if {[llength $repl]>2 ||\ + [lindex $repl 1]-[lindex $repl 0]>=0} then { + lappend replL $repl + set repl {0 -1} + } + lset repl 1 $l1 + incr l1 + lset repl 0 $l1 + } + } + if {[llength $repl]>2 || [lindex $repl 1]-[lindex $repl 0]>=0}\ + then {lappend replL $repl} + if {$matches} then { + lappend hunk [lsort -decreasing -integer -index 0 $replL] + lappend RL $hunk + } else { + lappend hunk "(-- did not match fromtext --)" + lappend log $hunk + } + } + foreach hunk $RL { + set applied 0 + set misapplied 0 + foreach repl [lindex $hunk 5] { + unset -nocomplain from to + for {set n [lindex $repl 1]} {$n>=[lindex $repl 0]}\ + {incr n -1} { + if {![info exists lift($n)]} then { + incr misapplied + continue + } elseif {![info exists from]} then { + set to [lindex $lift($n) 0] + set from $to + } elseif {[lindex $lift($n) 0] == $from-1} then { + set from [lindex $lift($n) 0] + } else { + set SL [lreplace $SL $from $to] + set to [lindex $lift($n) 0] + set from $to + } + incr applied + set n0 $n + } + if {[info exists from]} then { + set sprefix [lindex $lift($n0) 1] + set eprefix [lindex $lift($n0) 2] + } elseif {[info exists lift([lindex $repl 0])]} then { + foreach {from sprefix eprefix} $lift([lindex $repl 0])\ + break + set to [expr {$from-1}] + } else { + incr misapplied [llength [lrange $repl 2 end]] + continue + } + set eplen [string length $eprefix] + set epend [expr {$eplen-1}] + set cmd [list lreplace $SL $from $to] + foreach line [lrange $repl 2 end] { + if {$eprefix eq [string range $line 0 $epend]} then { + lappend cmd "$sprefix[string range $line $eplen end]" + } else { + lappend cmd $line + } + incr applied + } + set SL [eval $cmd] + } + if {$misapplied>0} then { + if {$applied>0} then { + lset hunk 5 "(-- was partially applied --)" + } else { + lset hunk 5 "(not applied)" + } + lappend log $hunk + } + } + set res "" + foreach hunk [lsort -index 0 -integer $log] { + foreach {start1 end1 start2 end2 lines msg} $hunk break + append res [format "@@ -%d,%d +%d,%d @@ %s\n"\ + $start1 [expr {$end1-$start1+1}]\ + $start2 [expr {$end2-$start2+1}] $msg] + foreach {type line} $lines { + switch -- $type 0 { + append res " " $line \n + } - - + { + append res $type $line \n + } + } + } + return $res +} +proc docstrip::util::thefile {fname args} { + set F [open $fname r] + if {[llength $args]} then { + if {[set code [ + catch {eval [linsert $args 0 fconfigure $F]} res + ]]} then { + close $F + return -code $code -errorinfo $::errorInfo -errorcode\ + $::errorCode + } + } + catch {read -nonewline $F} res + close $F + return $res +} +proc docstrip::util::import_unidiff {text {warnvar ""}} { + if {$warnvar ne ""} then {upvar 1 $warnvar warning} + set inheader 1 + set res [list] + set lines [list] + set end2 "not an integer" + foreach line [split $text \n] { + if {$inheader && [regexp {^(---|\+\+\+)} $line]}\ + then {continue} + switch -glob -- $line { *} { + lappend lines 0 [string range $line 1 end] + } {+*} { + lappend lines + [string range $line 1 end] + } {-*} { + lappend lines - [string range $line 1 end] + } @@* { + if {[string is integer $end2]} then { + lappend res [list $start1 $end1 $start2 $end2 $lines] + } + set len2 [set len1 ,1] + if {[ + regexp {^@@ -([0-9]+)(,[0-9]+)? \+([0-9]+)(,[0-9]+)? @@}\ + $line -> start1 len1 start2 len2 + ] && [scan "$start1 $len1,1" {%d ,%d} start1 len1]==2 &&\ + [scan "$start2 $len2,1" {%d ,%d} start2 len2]==2 + } then { + set end1 [expr {$start1+$len1-1}] + set end2 [expr {$start2+$len2-1}] + set inheader 0 + } else { + set end2 "not an integer" + append warning "Could not parse hunk header: " $line \n + } + set lines [list] + } "" { + } default { + append warning "Could not parse line: " $line \n + } + } + if {[string is integer $end2]} then { + lappend res [list $start1 $end1 $start2 $end2 $lines] + } + return $res +} +## +## +## End of file `docstrip_util.tcl'.
\ No newline at end of file diff --git a/tcllib/modules/docstrip/docstrip_util.test b/tcllib/modules/docstrip/docstrip_util.test new file mode 100644 index 0000000..5a7c1e0 --- /dev/null +++ b/tcllib/modules/docstrip/docstrip_util.test @@ -0,0 +1,84 @@ +## +## This is the file `docstrip_util.test', +## generated with the SAK utility +## (sak docstrip/regen). +## +## The original source files were: +## +## tcldocstrip.dtx (with options: `utiltest tcllibtest') +## +## In other words: +## ************************************** +## * This Source is not the True Source * +## ************************************** +## the true source is the file from which this one was generated. +## +source [file join\ + [file dirname [file dirname [file join [pwd] [info script]]]]\ + devtools testutilities.tcl] +testsNeedTcl 8.4 +testsNeedTcltest 2 +testing {useLocal docstrip.tcl docstrip} +testing {useLocal docstrip_util.tcl docstrip::util} +variable docstrip_sources_dir [localPath {}] +tcltest::testConstraint docstripSourcesAvailable [expr {[ + file exists [file join $docstrip_sources_dir docstrip.tcl] +] && [ + file exists [file join $docstrip_sources_dir tcldocstrip.dtx] +]}] +tcltest::test docstrip::util::thefile-1.1 {thefile without args}\ + -setup { + set Fname [tcltest::makeFile [ + join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + } \n + ] test.txt] +} -body { + docstrip::util::thefile $Fname +} -cleanup { + tcltest::removeFile $Fname +} -result [join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} +} \n] +tcltest::test docstrip::util::thefile-1.2 {thefile with wrong no. args}\ + -setup { + set Fname [tcltest::makeFile [ + join { + {% Just a minor test file (contents irrelevant).} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + } \n + ] test.txt] +} -body { + docstrip::util::thefile $Fname -translation binary -buffering +} -cleanup { + tcltest::removeFile $Fname +} -returnCodes error +tcltest::test docstrip::util::thefile-1.3 {thefile with args} -setup { + set Fname [tcltest::makeFile "Dummy content to overwrite" test.xxx] + set F [open $Fname w] + fconfigure $F -translation binary + puts -nonewline $F [encoding convertto utf-8 \u00E5\u00E4\u00F6] + close $F +} -body { + docstrip::util::thefile $Fname -encoding utf-8 +} -cleanup { + tcltest::removeFile $Fname +} -result \u00E5\u00E4\u00F6 +testsuiteCleanup +## +## +## End of file `docstrip_util.test'.
\ No newline at end of file diff --git a/tcllib/modules/docstrip/pkgIndex.tcl b/tcllib/modules/docstrip/pkgIndex.tcl new file mode 100644 index 0000000..2835539 --- /dev/null +++ b/tcllib/modules/docstrip/pkgIndex.tcl @@ -0,0 +1,23 @@ +## +## This is the file `pkgIndex.tcl', +## generated with the SAK utility +## (sak docstrip/regen). +## +## The original source files were: +## +## tcldocstrip.dtx (with options: `idx') +## +## In other words: +## ************************************** +## * This Source is not the True Source * +## ************************************** +## the true source is the file from which this one was generated. +## +if {![package vsatisfies [package provide Tcl] 8.4]} {return} +package ifneeded docstrip 1.2\ + [list source [file join $dir docstrip.tcl]] +package ifneeded docstrip::util 1.3.1\ + [list source [file join $dir docstrip_util.tcl]] +## +## +## End of file `pkgIndex.tcl'.
\ No newline at end of file diff --git a/tcllib/modules/docstrip/tcldocstrip.dtx b/tcllib/modules/docstrip/tcldocstrip.dtx new file mode 100644 index 0000000..ceb9c4c --- /dev/null +++ b/tcllib/modules/docstrip/tcldocstrip.dtx @@ -0,0 +1,4012 @@ +% +% \iffalse +%<*driver> +\documentclass{tclldoc} +\newenvironment{ttdescription}{% + \description + \def\makelabel##1{\hspace\labelsep\normalfont\ttfamily ##1}% +}{\enddescription} +\newcommand{\Tcllib}{\textsf{tcllib}} + +% The following is a hack, to get around some complications that +% arise when one tries to use the doc package on two levels +% simultaneously. Basically, I need to define my own macrocode +% environment, since at one place I'm going to need it for a block +% of code that contains the end-of-macrocode string +% "% \end{macrocode}" (note exactly four spaces). +\makeatletter +\newenvironment{Macrocode}{% + \macro@code\frenchspacing\@vobeyspaces\xMacro@code +}{\endmacrocode} +% The following piece of code uses the trick of having a macro grab +% some piece of material and thus ensure that it is tokenized under +% the expand-time catcodes, even though a different set of catcodes +% will be in force when that material is actually used. This saves +% from having to introduce an extra character of \catcode 0, and +% means only those characters which actually need it (i.e., those +% inside the final bracket group) are tokenized with unconventional +% catcodes. +\@firstofone{\bgroup + \def\@tempa#1{\egroup\def\xMacro@code##1#1{##1\end{Macrocode}}} + \catcode`\[=1\catcode`\]=2% + \catcode`\{=12\catcode`\}=12\catcode`\%=12% + \catcode`\\=13\catcode`\ =13\relax +\@tempa}[% \end{Macrocode}] +\makeatother +% An easier way around it would be to use a different number of +% spaces in that particular line, since hardly anyone would notice, +% but I want the details to be [emph correct]. + + +\begin{document} +\DocInput{tcldocstrip.dtx} +\end{document} +%</driver> +% \fi +% +% \title{The \textsf{docstrip} \Tcllogo\ package} +% \author{Lars Hellstr\"om} +% \date{25 August 2005} +% \maketitle +% +% \begin{abstract} +% The \textsf{docstrip} package provides a pure-\Tcllogo\ +% implementation of some of the functionality of the \LaTeX\ +% \textsc{docstrip} program. In particular, there is a command +% using which one can |source| \Tcllogo\ code from within a +% \texttt{.dtx} file. +% +% The \textsf{docstrip::util} package provides related +% functionality, which is more of interest at installation or +% development time than runtime. The main functionality areas are: +% (i)~hooks into the \Tcllogo\ package mechanisms, using which one +% can avoid depending on the \textsc{docstrip} program for +% \Tcllogo\ scripts; (ii)~statistical introspection into source +% files; (iii)~alternatives to \LaTeX\ as markup language; +% (iv)~patching of source files. +% \end{abstract} +% +% \changes{1.0}{2004/09/17}{Changing namespace to \texttt{docstrip} and +% also all command names. (LH)} +% +% \tableofcontents +% +% +% \section{Usage} +% +% \subsection{\textsf{docstrip} package} +% +% The simplest usage of the \textsf{docstrip} package is to source +% \Tcllogo\ code from within a \texttt{.dtx} file without having to +% generate any stripped file first. The command that does this is +% \describestring[proc][docstrip]{sourcefrom}|docstrip::sourcefrom|, +% which has the syntax +% \begin{quote} +% |docstrip::sourcefrom| \word{filename} \word{terminals} +% \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock} +% \end{quote} +% where \word{filename} is the source file name. The \word{terminals} +% is the list of guard expression terminals that should be considered +% true; the \textsc{docstrip} program calls these the ``options'' for +% the source file. The \word{option} and \word{value} arguments are +% passed on to |fconfigure|, to configure the file before |read|ing +% it. +% +% A typical usage is +% \begin{quote} +% |docstrip::sourcefrom foobar.dtx {foo debug}| +% \end{quote} +% which corresponds to |source|ing the file \texttt{temp.tcl} that +% would be generated by +% \begin{quote} +% |\generate{\file{temp.tcl}{\from{foobar.dtx}{foo,debug}}}| +% \end{quote} +% A more advanced usage (making use of the ability to |fconfigure| the +% source file before reading it) is +% \begin{quote} +% |docstrip::sourcefrom ruslish.dtx pkg -encoding utf-8| +% \end{quote} +% which ensures that the file is interpreted as being +% \texttt{utf-8} encoded. +% +% \iffalse +% (Files which require an encoding specification can actually be tricky +% to handle using the \textsc{docstrip} program, since most \TeX's will +% by default write \TeX-style |^^|-escapes for all characters outside +% visible ASCII, but the \textsf{docstrip} package handles such matters +% easily.) +% \fi +% +% The \textsf{docstrip} package can even be used in +% \texttt{pkgIndex.tcl} scripts. The typical pattern is\pagebreak[2] +%\begin{verbatim} +% package ifneeded foo 1.0 [format { +% package require docstrip +% docstrip::sourcefrom [file join %s foobar.dtx] foo +% } [list $dir]] +% package ifneeded bar 0.2 [format { +% package require docstrip +% docstrip::sourcefrom [file join %s foobar.dtx] bar +% } [list $dir]] +%\end{verbatim} +% where |format| is used to embed the package directory into the +% |package ifneeded| scripts; |list| provides the right amount of +% quoting of the directory string. Alternatively, one may use +% |docstrip::util::index_from_catalogue| (see below) to generate such +% scripts automatically. +% +% The semantics of |sourcefrom| closely follows those of |source|: The +% code is evaluated in the local context of the caller, a |return| will +% abort the sourcing early, and |info script| will return the +% \word{filename} for the duration of the |sourcefrom|. A difference is +% that |sourcefrom| does not stop at |\u001a| characters (control-Z, +% end of file) unless told to by an explicit |-eofchar| option. Also +% note that the entire file is ``docstripped'' +% before any of the code in it gets evaluated, so e.g. module nesting +% errors at the end of the file cannot be hidden by an early |return| +% in it. +% +% The actual ``docstripping'' is done by the +% \describestring[proc][docstrip]{extract}|docstrip::extract| command, +% which has the syntax +% \begin{quote} +% |docstrip::extract| \word{text} \word{terminals} +% \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% Unlike the \textsc{docstrip} program, which is file-oriented, this +% command takes the \word{text} to extract code from as an argument and +% returns the code that was extracted. The \word{terminals} is as for +% |sourcefrom| the list of guard expression terminals that should have +% the value true. +% +% The options are +% \begin{quote} +% |-annotate| \word{lines}\\ +% |-metaprefix| \word{string}\\ +% |-onerror| \begin{regblock}|throw|\regalt |puts|\regalt +% |ignore|\end{regblock}\\ +% |-trimlines| \word{boolean} +% \end{quote} +% These control some fine details of the extraction process. See +% Subsection~\ref{Ssec:Extract} for further information. +% +% The |extract| command does not as the \textsc{docstrip} program +% wrap the extracted code up with a preamble and postamble; it just +% handles the basic extraction, not the higher level operation of +% complete file generation. The +% |docstrip::util::modules_from_catalogue| command generates +% preambles and postambles, however. +% +% +% \subsection{\textsf{docstrip::util} package} +% +% The \textsf{docstrip::util} package is meant for collecting various +% utility procedures that may be useful for developers who make use of +% the \textsf{docstrip} package in some projects, either during +% development or during installation. It is separate from +% the main package to avoid overhead when |docstrip| is used in +% |package ifneeded| scripts. +% +% +% \subsubsection{Source file introspection} +% +% \describestring[proc][docstrip::util]{guards} +% The |guards| command collects information about the docstrip guards +% occurring in a file. It has the subcommands |names|, |counts|, +% |expressions|, |exprcounts|, and |exprmods| which return +% information about correct guards in various degrees of detail. The +% |exprerr| subcommand lists syntactically incorrect guard expressions, +% and the |rotten| subcommand lists the malformed guard lines. +% +% \describestring[proc][docstrip::util]{thefile} +% The |thefile| command is a conveniency for reading the contents of +% a file, since most other \texttt{docstrip::util} commands expect to +% be handed the text of \texttt{.dtx} or \texttt{.ddt} files as +% strings (or some other in-memory data structure). It takes as +% primary argument the name of the file to read, and will like +% |docstrip::sourcefrom| accept additional option--value pairs for +% configuring the file channel before reading from it. A final +% newline is dropped, so that the result can directly be |split| into +% a list of lines. +% +% +% \subsubsection{Package indexing with built-in catalogue} +% +% \changes{1.3}{2010/04/18}{Renamed the `directory' to `catalogue', +% to avoid overloading this term. Should be OK since so far only +% one project has contained a directory. (LH)} +% The basic method of installing a \Tcllogo\ package kept in a +% \texttt{.dtx} file is to run the corresponding \texttt{.ins} file +% to have \textsc{docstrip} (the program) generate one or several +% \texttt{.tcl} files, and then use the |pkg_mkIndex| to regenerate +% the package index file. This introduces a dependency on having +% \LaTeX\ available when installing however, so one might want to +% have a pure-\Tcllogo\ alternative. The \textsf{docstrip::util} +% package provides two: the |index_from_catalogue| and +% |modules_from_catalogue| commands. +% +% Parsing \texttt{.ins} files using \Tcllogo\ would be difficult, so +% the corresponding information about which docstrip modules make up +% a package is better put somewhere else. It turns out that it can +% easily be embedded into the \texttt{.dtx} file itself! By +% convention, the name of this module should be +% `|docstrip.tcl::catalogue|'---hinting both at who is expected to make +% use of this information and what it is. The contents of this module +% make up the \emph{catalogue} of the source file in question. +% +% \describestring[command]{pkgProvide} +% The main command to use in a catalogue is +% \begin{quote} +% |pkgProvide| \word{name} \word{version} \word{terminal-list} +% \end{quote} +% which means the module in this file which is selected by the +% terminals in \word{terminal-list} contains version \word{version} +% of the package named \word{name}. That module should not contain a +% |package provide| command, as one will be provided in the +% |package ifneeded| script. +% +% \describestring[command]{pkgIndex} +% An older alternative command for identifying package from within +% a catalogue is +% \begin{quote} +% |pkgIndex| \word{terminal}\regstar +% \end{quote} +% which means there is a module in the file which should be +% \emph{indexed} as a package; the package name and version is taken +% from the |package provide| command(s) found. The module is as usual +% defined by that the listed \word{terminal}s are true. +% +% \describestring[command]{fileoptions} +% A configuration command is +% \begin{quote} +% |fileoptions| \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% which sets the |fconfigure| options that will be used when reading +% the source file. Since the whole file was read into memory just to +% extract the catalogue, this command works by causing the file to be +% read again using a new set of options, and it thus has effect for +% the |pkgProvide| etc.\@ commands following it. (Even if it is +% perfectly legal, it would be rather strange to use |fileoptions| +% more than once in a file.) +% +% A catalogue for this file could be +% \begin{tcl} +%<*docstrip.tcl::catalogue> +pkgIndex pkg +pkgIndex utilpkg +%</docstrip.tcl::catalogue> +% \end{tcl} +% since the two packages include |package provide| commands. +% +% The |pkgProvide|, |pkgIndex|, and |fileoptions| commands are only +% available in catalogues. Unknown commands encountered in +% catalogues are silently ignored.\footnote{ +% This allows for future extensions of the catalogue, with commands +% that encode other kinds of entities. One application could be to +% list the files of a virtual file system, where the contents of +% the individual files are to be extracted from the source file. +% Another, perhaps more likely application would be to encode +% packages that span several source files. +% } +% +% \medskip +% +% \describestring[proc][docstrip::util]{modules_from_catalogue} +% The commands which look at the catalogue of a file are +% |index_from_catalogue| and |modules_from_catalogue|. The former +% appends |package ifneeded| commands (which make use of +% |docstrip::sourcefrom| rather than |source|) to a traditional +% \texttt{pkgIndex.tcl} file. The latter generates \texttt{.tm} files +% for the packages (overwriting previous files with the target +% names). It has the syntax +% \begin{quote} +% |docstrip::util::modules_from_catalogue| \word{target root} +% \word{source file} \begin{regblock}[\regstar] \word{option} +% \word{value} \end{regblock} +% \end{quote} +% where the \word{target root} is the directory used as starting +% point for the paths builts from package names, and +% \word{source file} is the file to process. The most common +% \word{option}s are: +% \begin{ttdescription} +% \item[-preamble] +% Message to put at the top of the generated file. Defaults to +% a space (which ends up contributing an empty line). +% \item[-postamble] +% Message to put at the bottom of the generated file. Defaults to +% being empty. +% \item[-options] +% \textsf{Docstrip} expressions terminals in addition to +% the basic \texttt{docstrip.tcl::catalogue} to use when +% extracting the catalogue. A sort of meta-configuration +% facility. +% \end{ttdescription} +% Traditionally, the |-preamble| would be used for a copyright +% message, but such messages can alternatively be embedded as +% ``metacomment lines''. +% +% \describestring[proc][docstrip::util]{index_from_catalogue} +% The syntax of |index_from_catalogue| is +% \begin{quote} +% |docstrip::util::index_from_catalogue| \word{directory} +% \word{pattern} \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% where \word{directory} is the directory whose \texttt{pkgIndex.tcl} +% file should be amended and \word{pattern} is a |glob|-pattern for +% files whose \texttt{docstrip.tcl::catalogue}s should be read. The most +% common \word{option}s are: +% \begin{ttdescription} +% \item[-recursein] +% If nonempty, then the operation will be repeated in each +% subdirectory matching the pattern specified as \word{value}. +% |-recursein *| causes the entire subtree rooted at |-root| to +% be processed. +% \item[-options] +% \textsf{Docstrip} expression terminals in addition to +% the basic \texttt{docstrip.tcl:\nolinebreak[1]:catalogue} to use +% when extracting the catalogue; a sort of metaconfiguration +% facility. +% \end{ttdescription} +% +% +% \subsubsection{Alternative markup languages} +% +% The |ddt2man| command provides an alternative to \LaTeX\ markup for +% programmers who think \LaTeX\ is too heavy (e.g.\ installation-wise) +% and prefer a pure-\Tcllogo\ documentation setup, namely to use +% \textsf{doctools}~\cite{doctools_fmt} man page markup. This is +% nowhere near as powerful as \LaTeX, but may well suffice in cases +% with less sophisticated typographical requirements. +% +% \describestring[proc][docstrip::util]{ddt2man} +% Since \textsf{doctools} cannot be configured to process +% docstrip-style master sources directly, a conversion to some format +% that can be processed is necessary, and that is precisely what the +% |ddt2man| command does. The syntax is +% \begin{quote} +% |docstrip::util::ddt2man| \word{ddt-text} +% \end{quote} +% where \word{ddt-text} is the contents of a master source code file +% and the result is the same text reformatted as \textsf{doctools} +% man page source. The command name comes from the recommended file +% suffixes: \textsf{doctools} man pages have the suffix \texttt{.man} +% and master source files with \textsf{doctools} markup in the comments +% should use the suffix \texttt{.ddt} to distinguish them from +% \texttt{.dtx} files which have \LaTeX\ markup in the comments. +% +% A typical usage might be +%\begin{verbatim} +%package require docstrip::util +%package require doctools +%doctools::new man2html -format html +%set ddt [docstrip::util::thefile somefile.ddt] +%set man [docstrip::util::ddt2man $ddt] +%set html [man2html format $man] +%\end{verbatim} +% after which the |html| variable contains ordinary HTML code. +% +% +% \subsubsection{Patching sources} +% +% \describestring[proc][docstrip::util]{patch} +% The |patch| command is a still slightly experimental utility for +% applying patches against extracted files to the master sources +% proper; it works by translating extracted file line numbers to +% master source file numbers and applies differences at the translated +% positions. Currently the text being patched is kept in memory as a +% list of lines, but this may change if this feature is more closely +% integrated with the \Tcllogo lib diff file utilies offered by the +% \textsf{rcs} package. |patch| also has a companion command +% \describestring[proc][docstrip::util]{import_unidiff} +% |import_unidiff| that translates patches to the format understood by +% the |patch| command. +% +% An example of using these commands would be +%\begin{verbatim} +%set sourceL [split [docstrip::util::thefile somefile.dtx] \n] +%set generated [docstrip::util::thefile foobar.tcl] +%set diff [docstrip::util::thefile foobar.patch] +%set conflicts [docstrip::util::patch sourceL {foo bar} $generated\ +% [docstrip::util::import_unidiff $diff]] +%\end{verbatim} +% after which one in principle can overwrite \texttt{somefile.dtx} +% with the result of |join $sourceL \n|, but more often one should +% rather send these patched contents to a text editor for further +% review. For one thing, there may be conflicts. For another, it is +% often necessary to also update the comment lines around modified +% sections. +% +% +% \section{Headers} +% +% The guiding principle for the various file headers has been to +% collect all occurrencies of a version number in the same place. +% This is not entirely possible, since the |manpage_begin| and +% |require| manpage commands both contain the package version +% numbers, but at least it is possible to collect |require|s, +% |package require|s, and |package provide|s together. +% \changes{1.3}{2010/04/30}{File headers interleaved, to simplify +% changing version numbers. (LH)} +% +% Since this leaves |manpage_begin| of the \textsf{doctools} manpages +% the odd man out, we'd better begin with that. +% \begin{tcl} +%<*man,utilman> +%<man>[manpage_begin docstrip n 1.2] +%<utilman>[manpage_begin docstrip_util n 1.3] +%<man>[see_also docstrip_util] +%<utilman>[see_also docstrip] +%<utilman>[see_also doctools] +%<utilman>[see_also doctools_fmt] +%<utilman>[keywords .ddt] +[keywords .dtx] +%<utilman>[keywords catalogue] +%<utilman>[keywords diff] +[keywords docstrip] +%<utilman>[keywords doctools] +[keywords documentation] +[keywords LaTeX] +[keywords {literate programming}] +%<utilman>[keywords module] +%<utilman>[keywords {package indexing}] +%<utilman>[keywords patch] +[keywords source] +%<utilman>[keywords {Tcl module}] +[copyright "2003\u20132010 Lars Hellstr\u00F6m\ + <Lars dot Hellstrom at residenset dot net>"] +[moddesc {Literate programming tool}] +%<man>[titledesc {Docstrip style source code extraction}] +%<utilman>[titledesc {Docstrip-related utilities}] +[category {Documentation tools}] +%</man,utilman> +% \end{tcl} +% The other files involved in this great interleaving are the actual +% packages (\Module{pkg} and \Module{utilpkg}) and a hardcoded index +% file (\Module{idx}). +% \changes{1.1}{2005/02/26}{Added \texttt{pkgIndex.tcl} source. +% (LH, after suggestion by AK)} +% +% \Tcllogo~8.4 is required because |info script| with an argument +% is used by the |sourcefrom| command, and there are also some uses +% of the |eq| operator in |if| expressions. +% \begin{tcl} +%<pkg,utilpkg>package require Tcl 8.4 +%<man,utilman>[require Tcl 8.4] +%<idx>if {![package vsatisfies [package provide Tcl] 8.4]} {return} +% \end{tcl} +% That version number check is the only part of the +% \texttt{pkgIndex.tcl} file that would not be done the same way by +% the standard |pkg_mkIndex| command. +% +% Next comes the \textsf{docstrip} package version. +% \begin{tcl} +%<pkg>package provide docstrip 1.2 +%<idx>package ifneeded docstrip 1.2\ +%<idx> [list source [file join $dir docstrip.tcl]] +%<man,utilman>[require docstrip [opt 1.2]] +%<utilpkg>package require docstrip 1.2 +% \end{tcl} +% The \textsf{docstrip::util} package has a dependency on the +% \textsf{docstrip} package, but not the other way around. Hence the +% next block is slightly shorter. +% \begin{tcl} +%<utilpkg>package provide docstrip::util 1.3.1 +%<idx>package ifneeded docstrip::util 1.3.1\ +%<idx> [list source [file join $dir docstrip_util.tcl]] +%<utilman>[require docstrip::util [opt 1.3]] +% \end{tcl} +% This ends the interleaved parts of the headers. +% +% The following is a trick to use non-ASCII characters in manpages +% without having to put them as such in the source: when an emdash +% (U+2014) is needed, just write |[vset emdash]|. +% \begin{tcl} +%<*man,utilman> +%<-ASCII>[vset emdash \u2014] +%<+ASCII>[vset emdash --] +[description] +%</man,utilman> +% \end{tcl} +% The \Module{ASCII} guard here makes it possible to fall back to +% simpler encodings, on platforms which require it, but the default +% is the proper emdash. +% +% The public commands in both packages are exported. This is +% meaningful mostly for the \textsf{docstrip::util} package, which +% imports |extract| from \textsf{docstrip}. The corresponding +% |namespace import| will however occur further down, to ensure that +% a combined file extracted with |pkg,utilpkg| works too. +% \changes{1.2}{2005/06/20}{Added namespace export code. (LH)} +% \begin{tcl} +%<*pkg> +namespace eval docstrip { + namespace export extract sourcefrom +} +%</pkg> +%<*utilpkg> +namespace eval docstrip::util { + namespace export ddt2man guard patch thefile\ + packages_provided index_from_catalogue modules_from_catalogue\ + classical_preamble classical_postamble +} +%</utilpkg> +% \end{tcl} +% +% \subsection{Test headers} +% +% What now remains to initialise are only the tests, but that is a +% slightly complicated affair, since it means interacting with the +% \Tcllib\ test harness. Originally the code didn't do that, +% so there is a \Module{tcllibtest} terminal which can be used to +% select whether to try it or not. +% +% The original route was to use the \textsf{tcltest} package provided +% by the \Tcllogo\ package mechanism, but explicitly |source| the +% \texttt{docstrip.tcl} file in the same directory as the +% \texttt{docstrip.test} being run. That meant you could have one +% \textsf{docstrip} version installed and run tests on another. +% \changes{1.2}{2005/09/18}{Introduced the +% \texttt{docstrip\_sources\_dir} variable as the directory in +% which to search for \texttt{docstrip.tcl}, +% \texttt{docstrip\_util.tcl}, and \texttt{tcldocstrip.dtx}. +% Using \texttt{file normalize} to compute it. (LH)} +% \begin{tcl} +%<*test,utiltest> +%<*!tcllibtest> +package require tcltest 2 +variable docstrip_sources_dir\ + [file dirname [file normalize [info script]]] +source [file join $docstrip_sources_dir docstrip.tcl] +puts "** Has Tcl docstrip package (v [package provide docstrip]) **" +%<*utiltest> +source [file join $docstrip_sources_dir docstrip_util.tcl] +puts "** Has Tcl docstrip::util package\ + (v [package provide docstrip::util]) **" +%</utiltest> +%</!tcllibtest> +% \end{tcl} +% +% The \Tcllib\ set-up instead begins with |source|ing +% \texttt{..\slash devtools\slash testutilities.tcl}, which +% (in that directory structure) will be a file that causes +% \textsf{tcltest} to be loaded. The other commands below are also +% provided by that file. +% \changes{1.2.1}{2006/09/13}{Modified the setup of the testsuite +% to match the other modules and packages in \Tcllib. (AK)} +% \begin{tcl} +%<*tcllibtest> +source [file join\ + [file dirname [file dirname [file join [pwd] [info script]]]]\ + devtools testutilities.tcl] +testsNeedTcl 8.4 +testsNeedTcltest 2 +testing {useLocal docstrip.tcl docstrip} +%<utiltest>testing {useLocal docstrip_util.tcl docstrip::util} +variable docstrip_sources_dir [localPath {}] +%</tcllibtest> +% \end{tcl} +% One of the tests require that \texttt{tcldocstrip.dtx} (this file) and +% \texttt{docstrip.tcl} are both present. A \textsf{tcltest} constraint +% is declared for this purpose. +% \begin{tcl} +tcltest::testConstraint docstripSourcesAvailable [expr {[ + file exists [file join $docstrip_sources_dir docstrip.tcl] +] && [ + file exists [file join $docstrip_sources_dir tcldocstrip.dtx] +]}] +%</test,utiltest> +% \end{tcl} +% +% +% \part{The docstrip package} +% +% \setnamespace{docstrip} +% +% Here follows the source both for the actual package and its manpage, +% the latter of which is in four sections: introduction, +% description of the format of files to be processed by +% \textsc{docstrip}, description of commands, and basic remarks on +% overall document structure. Since command descriptions and +% implementations appear in the same sections of the \texttt{.dtx} +% file, a big batch of manpage source has to appear first. +% +% +% \section{Manpage} +% +% The introduction is indended for \Tcllogo\ programmers who have not +% previously encountered \textsc{docstrip}---hence it is probably a +% bit boring for experienced \LaTeX\ programmers. +% \begin{macrocode} +%<*man> + +[syscmd Docstrip] is a tool created to support a brand of Literate +Programming. It is most common in the (La)TeX community, where it +is being used for pretty much everything from the LaTeX core and up, +but there is nothing about [syscmd docstrip] which prevents using it +for other types of software. +[para] + +In short, the basic principle of literate programming is that program +source should primarily be written and structured to suit the +developers (and advanced users who want to peek "under the hood"), not +to suit the whims of a compiler or corresponding source code consumer. +This means literate sources often need some kind of "translation" to an +illiterate form that dumb software can understand. +The [package docstrip] Tcl package handles this translation. +[para] + +Even for those who do not whole-hartedly subscribe to the philosophy +behind literate programming, [syscmd docstrip] can bring greater +clarity to in particular: +[list_begin itemized] + [item] programs employing non-obvious mathematics + [item] projects where separate pieces of code, perhaps in + different languages, need to be closely coordinated. +[list_end] +The first is by providing access to much more powerful typographical +features for source code comments than are possible in plain text. +The second is because all the separate pieces of code can be kept +next to each other in the same source file. +[para] + +The way it works is that the programmer edits directly only one or +several "master" source code files, from which [syscmd docstrip] +generates the more traditional "source" files compilers or the like +would expect. The master sources typically contain a large amount of +documentation of the code, sometimes even in places where the code +consumers would not allow any comments. The etymology of "docstrip" +is that this [emph doc]umentation was [emph strip]ped away (although +"code extraction" might be a better description, as it has always +been a matter of copying selected pieces of the master source rather +than deleting text from it). +The [package docstrip] Tcl package contains a reimplementation of +the basic extraction functionality from the [syscmd docstrip] +program, and thus makes it possible for a Tcl interpreter to read +and interpret the master source files directly. +[para] + +Readers who are not previously familiar with [syscmd docstrip] but +want to know more about it may consult the following sources. +[list_begin enumerated] +[enum] + [emph {The tclldoc package and class}], + [uri {http://ctan.org/tex-archive/macros/latex/contrib/tclldoc/}]. +[enum] + [emph {The DocStrip utility}], + [uri {http://ctan.org/tex-archive/macros/latex/base/docstrip.dtx}]. +[enum] + [emph {The doc and shortvrb Packages}], + [uri {http://ctan.org/tex-archive/macros/latex/base/doc.dtx}]. +[enum] + Chapter 14 of + [emph {The LaTeX Companion}] (second edition), + Addison-Wesley, 2004; ISBN 0-201-36299-6. +[list_end] +% \end{macrocode} +% +% \subsection{File format} +% +% In order to keep some kind of document structure in this file, it is +% best that the manpage sections are present also in the \LaTeX\ table +% of contents. +% +% \begin{macrocode} + +[section {File format}] + +The basic unit [syscmd docstrip] operates on are the [emph lines] of +a master source file. Extraction consists of selecting some of these +lines to be copied from input text to output text. The basic +distinction is that between [emph {code lines}] (which are copied and +do not begin with a percent character) and [emph {comment lines}] +(which begin with a percent character and are not copied). + +[example { +%</man> +% \end{macrocode} +% +% At this point, let's do a little trick: use this example also as the +% first test. This is just a matter of putting groups of lines in the +% right modules. +% \begin{tcl} +%<*test> +tcltest::test docstrip-1.1 {code/comment line distinction} -body { +%</test> +%<*test,man> + docstrip::extract [join { + {% comment} + {% more comment !"#$%&/(} + {some command} + { % blah $blah "Not a comment."} + {% abc; this is comment} + {# def; this is code} + {ghi} + {% jkl} + } \n] {} +%<man>}] +%<man>returns the same sequence of lines as +%<man>[example { +%<test>} -result [ + join { + {some command} + { % blah $blah "Not a comment."} + {# def; this is code} + {ghi} "" + } \n +%<test>] +%</test,man> +% \end{tcl} +% This completes the code for the test, so let's switch back to just +% \Module{man}. +% \begin{macrocode} +%<*man> +}] + +It does not matter to [syscmd docstrip] what format is used for the +documentation in the comment lines, but in order to do better than +plain text comments, one typically uses some markup language. Most +commonly LaTeX is used, as that is a very established standard and +also provides the best support for mathematical formulae, but the +[package docstrip::util] package also gives some support for +[term doctools]-like markup. +[para] + +Besides the basic code and comment lines, there are also +[emph {guard lines}], which begin with the two characters '%<', and +[emph {meta-comment lines}], which begin with the two characters +'%%'. Within guard lines there is furthermore the distinction between +[emph {verbatim guard lines}], which begin with '%<<', and ordinary +guard lines, where the '%<' is not followed by another '<'. The last +category is by far the most common. +[para] + +Ordinary guard lines conditions extraction of the code line(s) they +guard by the value of a boolean expression; the guarded block of +code lines will only be included if the expression evaluates to true. +The syntax of an ordinary guard line is one of +[example { + '%' '<' STARSLASH EXPRESSION '>' + '%' '<' PLUSMINUS EXPRESSION '>' CODE +}] +where +[example { + STARSLASH ::= '*' | '/' + PLUSMINUS ::= | '+' | '-' + EXPRESSION ::= SECONDARY | SECONDARY ',' EXPRESSION + | SECONDARY '|' EXPRESSION + SECONDARY ::= PRIMARY | PRIMARY '&' SECONDARY + PRIMARY ::= TERMINAL | '!' PRIMARY | '(' EXPRESSION ')' + CODE ::= { any character except end-of-line } +}] +Comma and vertical bar both denote 'or'. Ampersand denotes 'and'. +Exclamation mark denotes 'not'. A TERMINAL can be any nonempty string +of characters not containing '>', '&', '|', comma, '(', or ')', +although the [syscmd docstrip] manual is a bit restrictive and only +guarantees proper operation for strings of letters (although even +the LaTeX core sources make heavy use also of digits in TERMINALs). +The second argument of [cmd docstrip::extract] is the list of those +TERMINALs that should count as having the value 'true'; all other +TERMINALs count as being 'false' when guard expressions are evaluated. +[para] + +In the case of a '%<*[emph EXPRESSION]>' guard, the lines guarded are +all lines up to the next '%</[emph EXPRESSION]>' guard with the same +[emph EXPRESSION] (compared as strings). The blocks of code delimited +by such '*' and '/' guard lines must be properly nested. +% \end{macrocode} +% This looks like a good place for another example. +% \begin{tcl} +[example { +%</man> +%<*man,test> +%<test>tcltest::test docstrip-1.2 {blocks and nesting} -body { + set text [join { + {begin} + {%<*foo>} + {1} + {%<*bar>} + {2} + {%</bar>} + {%<*!bar>} + {3} + {%</!bar>} + {4} + {%</foo>} + {5} + {%<*bar>} + {6} + {%</bar>} + {end} + } \n] + set res [docstrip::extract $text foo] + append res [docstrip::extract $text {foo bar}] + append res [docstrip::extract $text bar] +%<*man> +}] +sets $res to the result of +[example { +%</man> +%<test>} -result [ + join { + {begin} + {1} + {3} + {4} + {5} + {end} + {begin} + {1} + {2} + {4} + {5} + {6} + {end} + {begin} + {5} + {6} + {end} "" + } \n +%<test>] +%</man,test> +%<*man> +}] +% \end{tcl} +% \begin{macrocode} + +In guard lines without a '*', '/', '+', or '-' modifier after the +'%<', the guard applies only to the CODE following the '>' on that +single line. A '+' modifier is equivalent to no modifier. A '-' +modifier is like the case with no modifier, but the expression is +implicitly negated, i.e., the CODE of a '%<-' guard line is only +included if the expression evaluates to false. +[para] + +Metacomment lines are "comment lines which should not be stripped +away", but be extracted like code lines; these are sometimes used for +copyright notices and similar material. The '%%' prefix is however +not kept, but substituted by the current [option -metaprefix], which +is customarily set to some "comment until end of line" character (or +character sequence) of the language of the code being extracted. +% \end{macrocode} +% Ho hum, another example\slash test. +% \begin{macrocode} +[example { +%</man> +%<*man,test> +%<*test> +tcltest::test docstrip-1.3 {plusminus guards and metacomments} -body { +%</test> + set text [join { + {begin} + {%<foo> foo} + {%<+foo>plusfoo} + {%<-foo>minusfoo} + {middle} + {%% some metacomment} + {%<*foo>} + {%%another metacomment} + {%</foo>} + {end} + } \n] + set res [docstrip::extract $text foo -metaprefix {# }] + append res [docstrip::extract $text bar -metaprefix {#}] +%<*man> +}] +sets $res to the result of +[example { +%</man> +%<test>} -result [ + join { + {begin} + { foo} + {plusfoo} + {middle} + {# some metacomment} + {# another metacomment} + {end} + {begin} + {minusfoo} + {middle} + {# some metacomment} + {end} "" + } \n +%<test>] +%</man,test> +%<*man> +}] + +Verbatim guards can be used to force code line +interpretation of a block of lines even if some of them happen to look +like any other type of lines to docstrip. A verbatim guard has the +form '%<<[emph END-TAG]' and the verbatim block is terminated by the +first line that is exactly '%[emph END-TAG]'. +[example { +%</man> +%<*man,test> +%<*test> +tcltest::test docstrip-1.4 {verbatim mode} -body { +%</test> + set text [join { + {begin} + {%<*myblock>} + {some stupid()} + { #computer<program>} + {%<<QQQ-98765} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + {%QQQ-98765} + { using*strange@programming<language>} + {%</myblock>} + {end} + } \n] + set res [docstrip::extract $text myblock -metaprefix {# }] + append res [docstrip::extract $text {}] +%<*man> +}] +sets $res to the result of +[example { +%</man> +%<test>} -result [ + join { + {begin} + {some stupid()} + { #computer<program>} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + { using*strange@programming<language>} + {end} + {begin} + {end} "" + } \n +%<test>] +%</man,test> +%<*man> +}] +The processing of verbatim guards takes place also inside blocks of +lines which due to some outer block guard will not be copied. +[para] + +The final piece of [syscmd docstrip] syntax is that extraction +stops at a line that is exactly "\endinput"; this is often used to +avoid copying random whitespace at the end of a file. In the unlikely +case that one wants such a code line, one can protect it with a +verbatim guard. +% \end{macrocode} +% Thus far the general descriptions; now for the actual commands. +% The manpage source for these are next to the actual implementations. +% \begin{macrocode} + +[section Commands] + +The package defines two commands. + +[list_begin definitions] +% \end{macrocode} +% +% \section{Command implementations} +% +% \subsection{Code extraction} +% \label{Ssec:Extract} +% +% \begin{proc}{extract} +% The |extract| procedure implements the core functionality of the +% \textsc{docstrip} program: copying the some lines of code as +% directed by relevant guard linnes. The main difference is that this +% takes the input as a string and returns output as a string. Each +% line in the return value ends with a newline. +% +% The syntax is +% \begin{quote} +% |docstrip::extract| \word{text} \word{terminal list} +% \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% where \word{text} is the string to docstrip and \word{terminal list} +% is the list of expression terminals that should be true. +% \changes{1.0}{2004/09/30}{Switched to option--value syntax for +% equivalents of \textsc{docstrip} parameters. (LH)} +% The options are +% \begin{quote} +% |-annotate| \word{lines}\\ +% |-metaprefix| \word{string}\\ +% |-onerror| \begin{regblock}|throw|\regalt |puts|\regalt +% |ignore|\end{regblock}\\ +% |-trimlines| \word{boolean} +% \end{quote} +% \changes{1.2}{2005/06/16}{Added \texttt{-annotate} option. (LH)} +% +% The \describeopt[docstrip]{extract}{-metaprefix}|-metaprefix| value +% is the string to use for the \textsc{docstrip} parameter +% \verb|\MetaPrefix|. The default is `|%%|'. +% The \describeopt[docstrip]{extract}{-trimlines}|-trimlines| option +% specifies whether spaces at the end of a line should be trimmed +% away before it is processed. For compatibility with +% \textsc{docstrip} (which due to a quirk in the low-level input +% routines of \TeX\ cannot help doing this), this is by default on. +% +% The \describeopt[docstrip]{extract}{-annotate}|-annotate| option +% modifies the output format, so that each extracted line is followed +% by \word{lines} lines of annotation information. These extra lines +% have the following format +% \begin{quote} +% \word{type} \word{offprefix} \word{onprefix}\\ +% \meta{lineno}\\ +% \meta{current stack} +% \end{quote} +% If \word{lines} is |0| then none of the above lines is included. If +% \word{lines} is |1| then only the first line is included. If +% \word{lines} is |2| then the first two lines are included. Finally +% if \word{lines} is |3| then all three lines are included. The +% behaviour for other values of \word{lines} is unspecified. The +% default value is |0|. +% +% A first annotation line is a list of three elements. The first +% element is a ``line type'', the second element is the prefix string +% that was removed from the line (an empty string if nothing was +% removed), and the third element is the prefix that was added to the +% line (either the |-metaprefix| value or an empty string). The line +% type is one of: |V|~(verbatim), |M|~(metacomment), |+|~(plus or no +% modifier guard line), |-|~(minus modifier guard line), and +% |.|~(normal line). The second annotation line is simply the current +% input line number. The third annotation line is the current block +% guard stack---a list of guard expression strings. +% +% The \describeopt[docstrip]{extract}{-onerror}|-onerror| option +% specifies what should happen when an error in the \word{text} being +% processed is detected. The value |puts| causes error messages to +% be written to |stderr|, but processing continues. |ignore| causes +% processing to continue silently. The default |throw| causes a +% \Tcllogo\ error to be thrown. In this last case, the |errorCode| is +% set to a list with the format +% \begin{quote} +% |DOCSTRIP| \word{situation} \word{lineno} +% \end{quote} +% where \word{lineno} is the line number (starting at one) of the line +% where the error was detected. The \word{situation}s are described +% below, at the positions in the code where they are detected. +% +% Now, for the manpage, a quick resum\'e of the above. +% \begin{macrocode} +[call [cmd docstrip::extract] [arg text] [arg terminals] [ + opt "[arg option] [arg value] ..." +]] + The [cmd extract] command docstrips the [arg text] and returns the + extracted lines of code, as a string with each line terminated with + a newline. The [arg terminals] is the list of those guard + expression terminals which should evaluate to true. + The available options are: + [list_begin options] + [opt_def -annotate [arg lines]] + Requests the specified number of lines of annotation to follow + each extracted line in the result. Defaults to 0. Annotation lines + are mostly useful when the extracted lines are to undergo some + further transformation. A first annotation line is a list of three + elements: line type, prefix removed in extraction, and prefix + inserted in extraction. The line type is one of: 'V' (verbatim), + 'M' (metacomment), '+' (+ or no modifier guard line), '-' (- + modifier guard line), '.' (normal line). A second annotation line + is the source line number. A third annotation line is the current + stack of block guards. Requesting more than three lines of + annotation is currently not supported. + [opt_def -metaprefix [arg string]] + The string by which the '%%' prefix of a metacomment line will + be replaced. Defaults to '%%'. For Tcl code this would typically + be '#'. + [opt_def -onerror [arg keyword]] + Controls what will be done when a format error in the [arg text] + being processed is detected. The settings are: + [list_begin definitions] + [def [const ignore]] + Just ignore the error; continue as if nothing happened. + [def [const puts]] + Write an error message to [const stderr], then continue + processing. + [def [const throw]] + Throw an error. The [option -errorcode] is set to a list whose + first element is [const DOCSTRIP], second element is the + type of error, and third element is the line number where + the error is detected. This is the default. + [list_end] + [opt_def -trimlines [arg boolean]] + Controls whether [emph spaces] at the end of a line should be + trimmed away before the line is processed. Defaults to true. + [list_end] + + It should be remarked that the [arg terminals] are often called + "options" in the context of the [syscmd docstrip] program, since + these specify which optional code fragments should be included. + +%</man> +% \end{macrocode} +% Hmm\dots\ Perhaps not so quick, after all. +% \begin{tcl} +%<*pkg> +proc docstrip::extract {text terminals args} { + array set O { + -annotate 0 + -metaprefix %% + -onerror throw + -trimlines 1 + } + array set O $args +% \end{tcl} +% The |O| array is for options of this procedure. The |T| array is +% for the terminals, so that the truth value of a terminal can be +% tested using |info exists|. +% \begin{tcl} + foreach t $terminals {set T($t) ""} +% \end{tcl} +% |stripped| is where the text that passes docstripping is collected. +% \begin{tcl} + set stripped "" +% \end{tcl} +% |block_stack| is the list of modules inside which the current line +% lies. |offlevel| is the number of modules that must be exited +% before code lines should once again be included. |verbatim| is a +% flag for whether verbatim mode is in force. +% \begin{tcl} + set block_stack [list] + set offlevel 0 + set verbatim 0 +% \end{tcl} +% |lineno| is the input line number counter, for use in error +% messages. The first line in the file has number |1|. +% \begin{tcl} + set lineno 0 +% \end{tcl} +% Here starts the main loop over lines in the \word{text}. It +% constitutes the majority of the procedure and is split in two +% parts. The smaller part handles lines in verbatim mode (unusual), +% the large part handles lines in normal mode (with comment lines, +% code lines, guard lines, and so on). |continue| is being used in +% this loop to skip generation of annotation lines, for those branches +% that do not contribute a line to the output in the first place. +% \begin{tcl} + foreach line [split $text \n] { + incr lineno + if {$O(-trimlines)} then { + set line [string trimright $line " "] + } + if {$verbatim} then { + if {$line eq $endverbline} then { + set verbatim 0 + continue + } elseif {$offlevel} then { + continue + } + append stripped $line \n + if {$O(-annotate)>=1} then {append stripped {V "" ""} \n} + } else { +% \end{tcl} +% Here starts the processing of lines in non-verbatim mode. +% \begin{tcl} + switch -glob -- $line %%* { + if {!$offlevel} then { + append stripped $O(-metaprefix)\ + [string range $line 2 end] \n + if {$O(-annotate)>=1} then { + append stripped [list M %% $O(-metaprefix)] \n + } + } + } %<<* { + set endverbline "%[string range $line 3 end]" + set verbatim 1 + continue + } %<* { +% \end{tcl} +% This is the case of an ordinary guard line, which accounts for most +% of the complexities in the file format. Here one can also encounter +% a number of conditions which constitute errors in the data being +% processed. The first of these is the +% \describestring[error situation]{BADGUARD}|BADGUARD| +% \word{situation}: the line looks like a guard line, but there is no +% |>| terminating the guard expression. +% \begin{tcl} + if {![ + regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\ + modifier expression line + ]} then { + extract,error BADGUARD\ + "Malformed guard \"\n$line\n\"" + "Malformed guard on line $lineno" + continue + } +% \end{tcl} +% At this point, an ordinary guard line has successfully been split +% into parts. First the expression is evaluated, by converting it +% to an |expr| expression. +% \begin{tcl} + regsub -all -- {\\|\{|\}|\$|\[|\]| |;} $expression\ + {\\&} E + regsub -all -- {,} $E {|} E + regsub -all -- {[^()|&!]+} $E {[info exists T(&)]} E + if {[catch {expr $E} val]} then { + extract,error EXPRERR\ + "Error in expression <$expression> ignored"\ + "docstrip: $val" + set val -1 + } +% \end{tcl} +% If |$E| isn't a valid |expr| expression, then the original guard +% expression must have been malformed. That is an +% \describestring[error situation]{EXPRERR}|EXPRERR| \word{situation}. +% \changes{1.0}{2004/09/29}{Catching errors in expressions. (LH)} +% +% With the expression evaluated, the processing of a guard line +% now branches according to its type. +% \begin{tcl} + switch -exact -- $modifier * { + lappend block_stack $expression + if {$offlevel || !$val} then {incr offlevel} + continue + } / { + if {![llength $block_stack]} then { +% \end{tcl} +% In this case there was no open block for this guard to end. That +% is a \describestring[error situation]{SPURIOUS}|SPURIOUS| +% \word{situation}. +% \begin{tcl} + extract,error SPURIOUS\ + "Spurious end block </$expression> ignored"\ + "Spurious end block </$expression>" + } else { + if {[string compare $expression\ + [lindex $block_stack end]]} then { +% \end{tcl} +% In this case the expression of the block being closed does not match +% the expression on the block on top of the stack. That is a +% \describestring[error situation]{MISMATCH}|MISMATCH| +% \word{situation}. \textsc{docstrip} by default raises an error and +% recovers by treating this situation as a typo. +% \begin{tcl} + extract,error MISMATCH\ + "Found </$expression> instead of\ + </[lindex $block_stack end]>" + } +% \end{tcl} +% All that error processing makes it easy to lose track, but the +% following two lines are what does the real work for an end of block +% guard: pop a block off the stack and decrement the |offlevel|. +% \begin{tcl} + if {$offlevel} then {incr offlevel -1} + set block_stack [lreplace $block_stack end end] + } + continue +% \end{tcl} +% These last cases of the |switch| handle |-|, |+|, and ``no +% modifier'' lines. +% \begin{tcl} + } - { + if {$offlevel || $val} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then { + append stripped [list - %<-${expression}> ""] \n + } + } default { + if {$offlevel || !$val} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then { + append stripped\ + [list + %<${modifier}${expression}> ""] \n + } + } + } %* {continue}\ +% \end{tcl} +% Back to the outer |switch|. With comment lines, nothing is done. +% A line being the exact string |\endinput| terminates the stripping. +% \begin{tcl} + {\\endinput} { + break + } default { +% \end{tcl} +% Other lines are code lines. These are included or not, depending on +% the |offlevel|. +% \begin{tcl} + if {$offlevel} then {continue} + append stripped $line \n + if {$O(-annotate)>=1} then {append stripped {. "" ""} \n} + } + } +% \end{tcl} +% Finally there is the code for annotation lines two and above. +% \begin{tcl} + if {$O(-annotate)>=2} then {append stripped $lineno \n} + if {$O(-annotate)>=3} then {append stripped $block_stack \n} + } + return $stripped +} +% \end{tcl} +% +% \begin{proc}{extract,error} +% Since the |extract| procedure can detect many different +% errors which should all go through roughtly the same handling, +% the common parts of that have been factored out into this +% |extract,error| procedure. It accesses the variable |lineno| and +% array element |O(-onerror)| in the local context of its caller +% to determine the current line number and error reporting mode. +% Apart from that, the call syntax is +% \begin{quote} +% |docstrip::extract,error| \word{situation} \word{message} +% \word{error message}\regopt +% \end{quote} +% where \word{situation} is what would be used to identify the +% error in |errorCode| (if |-onerror| is |throw|), \word{message} +% is the message that would be written to |stderr| (if |-onerror| +% is |puts|), and \word{error message} is the error message to use +% (if |-onerror| is |throw|). The default for \word{error message} +% is the \word{message}. Neither \word{message} nor \word{error +% message} should end with a period, as such punctuation may be +% provided by |extract,error|. +% \changes{1.1}{2005/02/27}{Procedure factored out from +% \texttt{extract}, as suggested by AK. (LH)} +% \begin{tcl} +proc docstrip::extract,error {situation message {errmessage ""}} { + upvar 1 O(-onerror) onerror lineno lineno + switch -- [string tolower $onerror] "puts" { + puts stderr "docstrip: $message on line $lineno." + } "ignore" {} default { + if {$errmessage ne ""} then { + error $errmessage "" [list DOCSTRIP $situation $lineno] + } else { + error $message "" [list DOCSTRIP $situation $lineno] + } + } +} +%</pkg> +% \end{tcl} +% \end{proc} +% \end{proc} +% +% The following tests annotation. It is mostly the same code as in the +% verbatim mode test. +% \begin{tcl} +%<*test> +tcltest::test docstrip-1.5 {annotation} -body { + set text [join { + {begin} + {%<*myblock>} + {some stupid()} + {%<foo> #computer<program>} + {%<<QQQ-98765} + {% These three lines are copied verbatim (including percents} + {%% even if -metaprefix is something different than %%).} + {%</myblock>} + {%QQQ-98765} + { using*strange@programming<language>} + {%</myblock>} + {%%end} + } \n] + docstrip::extract $text {myblock foo} -metaprefix {# } -annotate 3 +} -result [ + join { + {begin} {. "" ""} 1 {} + {some stupid()} {. "" ""} 3 myblock + { #computer<program>} {+ %<foo> {}} 4 myblock + {% These three lines are copied verbatim (including percents} + {V "" ""} 6 myblock + {%% even if -metaprefix is something different than %%).} + {V "" ""} 7 myblock + {%</myblock>} {V "" ""} 8 myblock + { using*strange@programming<language>} {. "" ""} 10 myblock + {# end} {M %% {# }} 12 {} + "" + } \n +] +% \end{tcl} +% +% The following is a test of the |extract| procedure, which compares its +% output to the \textsc{docstrip} program output. If need be, and \LaTeX\ +% is not available, then this could also be modified to produce a new +% version of \texttt{docstrip.tcl} using the |extract| command of +% an older version. +% \begin{tcl} +tcltest::test docstrip-2.1 {have docstrip extract itself} -constraints { + docstripSourcesAvailable +} -body { + # First read in the ready-stripped file, but gobble the preamble and + # postamble, as those are a bit messy to reproduce. + set F [open [file join $docstrip_sources_dir docstrip.tcl] r] + regsub -all -- {(^|\n)#[^\n]*} [read $F] {} stripped + close $F + # Then read the master source and strip it manually. + set F [open [file join $docstrip_sources_dir tcldocstrip.dtx] r] + set source [read $F] + close $F + set stripped2 [docstrip::extract $source pkg -metaprefix ##] + # Finally compare the two. + if {[string trim $stripped \n] ne [string trim $stripped2 \n]} then { + error "$strippped\n ne \n$stripped2" + } +} +%</test> +% \end{tcl} +% +% +% +% \subsection{Code sourcing} +% +% \begin{proc}{sourcefrom} +% This procedure behaves as a docstripping |source| command: it reads +% a file, docstrips its contents in memory, and evaluates the result +% as a \Tcllogo\ script in the context of the caller. The syntax is +% \begin{quote} +% |docstrip::sourcefrom| \word{filename} \word{terminals} +% \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock} +% \end{quote} +% where \word{filename} is the file name and \word{terminals} is the +% list of true guard expression terminals. The \word{option} and +% \word{value} arguments are passed on to |fconfigure|, to configure +% the file before |read|ing it. +% \changes{1.0}{2004/10/01}{Added \texttt{info script} management. +% (LH)} +% \begin{tcl} +%<*man> +[call [cmd docstrip::sourcefrom] [arg filename] [arg terminals] [ + opt "[arg option] [arg value] ..." +]] + The [cmd sourcefrom] command is a docstripping emulation of + [cmd source]. It opens the file [arg filename], reads it, closes it, + docstrips the contents as specified by the [arg terminals], and + evaluates the result in the local context of the caller, during + which time the [cmd info] [method script] value will be the + [arg filename]. The options are passed on to [cmd fconfigure] to + configure the file before its contents are read. The + [option -metaprefix] is set to '#', all other [cmd extract] + options have their default values. +%</man> +%<*pkg> +proc docstrip::sourcefrom {name terminals args} { + set F [open $name r] + if {[llength $args]} then { + eval [linsert $args 0 fconfigure $F] + } + set text [read $F] + close $F + set oldscr [info script] + info script $name + set code [catch { + uplevel 1 [extract $text $terminals -metaprefix #] + } res] + info script $oldscr + if {$code == 1} then { + error $res $::errorInfo $::errorCode + } else { + return $res + } +} +%</pkg> +% \end{tcl} +% \end{proc} +% +% Testing the above procedure requires an external file. The business +% with |info script| is to check that this is getting set and reset +% correctly. The business with the |baz| variable tests that the file +% contents are being evaluated in the context calling |sourcefrom|. +% \changes{1.2}{2005/10/02}{Moddified test to make it work when +% tmpdir is not the current directory. (LH)} +% \begin{tcl} +%<*test> +tcltest::test docstrip-2.2 {soucefrom} -setup { + set dtxname [tcltest::makeFile [join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + {set baz 1} + {%</foo>} + {%<-foo>return} + {%</bar>} + {puts $baz} + {puts [file tail [info script]]} + {%<*!foo>} + {puts C} + "%% Tricky comment; guess what comes next\\" + {%</!foo>} + {incr baz} +% \end{tcl} +% What the above construction does depends on the truth value of |foo|. +% When true, the \Module{!foo} block is skipped in its entirety, and +% thus the next command after |puts [file tail [info script]]| is +% |incr baz|. However when |foo| is false the block will be included. +% The metacomment line gets a prefix |#| and will therefore become +% a comment when the code is evaluated. The backslash escapes the +% subsequent newline, and thus the |incr baz| will only be part of +% a \Tcllogo\ comment. +% \begin{tcl} + {puts "baz=$baz"} + } \n] te27st01.dtx] +} -body { + set baz 0 + puts [info script] + docstrip::sourcefrom $dtxname {foo bar} + puts [info script] + docstrip::sourcefrom $dtxname {} + docstrip::sourcefrom $dtxname {bar} + puts $baz +} -cleanup { + tcltest::removeFile $dtxname +} -output [join [list\ + [info script]\ + {A} {B} {1} {1} {te27st01.dtx} {baz=2}\ + [info script]\ + {A} {2} {te27st01.dtx} {C} {baz=2}\ + {A} {B}\ + {2} "" +] \n] +%</test> +% \end{tcl} +% +% +% +% \section{Manpage section on document structure} +% +% This completes the package code, but there are more things which +% should be said on the manpage. +% +% +% \begin{macrocode} +%<*man> +[list_end] + + +[section {Document structure}] + +The file format (as described above) determines whether a master +source code file can be processed correctly by [syscmd docstrip], +but the usefulness of the format is to no little part also dependent +on that the code and comment lines together constitute a well-formed +document. +[para] + +For a document format that does not require any non-Tcl software, see +the [cmd ddt2man] command in the [package docstrip::util] package. It +is suggested that files employing that document format are given the +suffix [file .ddt], to distinguish them from the more traditional +LaTeX-based [file .dtx] files. +[para] + +Master source files with [file .dtx] extension are usually set up so +that they can be typeset directly by [syscmd latex] without any +support from other files. This is achieved by beginning the file +with the lines +[example_begin] + % \iffalse + %<*driver> + \documentclass{tclldoc} + \begin{document} + \DocInput{[emph filename.dtx]} + \end{document} + %</driver> + % \fi +[example_end] +or some variation thereof. The trick is that the file gets read twice. +With normal LaTeX reading rules, the first two lines are comments and +therefore ignored. The third line is the document preamble, the fourth +line begins the document body, and the sixth line ends the document, +so LaTeX stops there [vset emdash] non-comments below that point in +the file are never subjected to the normal LaTeX reading rules. Before +that, however, the \DocInput command on the fifth line is processed, +and that does two things: it changes the interpretation of '%' from +"comment" to "ignored", and it inputs the file specified in the +argument (which is normally the name of the file the command is in). +It is this second time that the file is being read that the comments +and code in it are typeset. +[para] + +The function of the \iffalse ... \fi is to skip lines two to seven +on this second time through; this is similar to the "if 0 { ... }" +idiom for block comments in Tcl code, and it is needed here because +(amongst other things) the \documentclass command may only be +executed once. The function of the <driver> guards is to prevent this +short piece of LaTeX code from being extracted by [syscmd docstrip]. +The total effect is that the file can function both as a LaTeX +document and as a [syscmd docstrip] master source code file. +[para] + +It is not necessary to use the tclldoc document class, but that does +provide a number of features that are convenient for [file .dtx] +files containing Tcl code. More information on this matter can be +found in the references above. + +%</man> +% \end{macrocode} +% +% +% \part{The docstrip utilities package} +% +% The |extract| command is used by several \textsf{docstrip::util} +% commands, so it is imported. +% \begin{tcl} +%<*utilpkg> +namespace eval docstrip::util { + namespace import [namespace parent]::extract +} +%</utilpkg> +% \end{tcl} +% \setnamespace{docstrip::util} +% +% \begin{macrocode} +%<*utilman> +The [package docstrip::util] package is meant for collecting various +utility procedures that are mainly useful at installation or +development time. It is separate from the base package to avoid +overhead when the latter is used to [cmd source] code. +[para] +%</utilman> +% \end{macrocode} +% +% \section{Package indexing and generation} +% +% Manually writing \texttt{pkgIndex.tcl} files for +% \textsf{docstrip}-encoded packages gets boring after a while +% (especially since they will have to be updated after every version +% increment), so one would like to automate this task. The following +% implements a mechanism for this that parallels the standard +% |pkg_mkIndex| command. +% +% +% \subsection{The catalogue} +% +% The main difference between a \textsf{docstrip} file and an +% ordinary \texttt{.tcl} file is that it is not clear to the casual +% reader what modules in a file should be combined to make a directly +% sourceable file. This information can however be encoded as a +% separate module into the source file itself. +% +% The special module holding catalogue information will be +% \begin{quote} +% \Module{docstrip.tcl::catalogue} +% \end{quote} +% The \texttt{docstrip.tcl} prefix here is intended as a clear +% indication of who is meant to read this information. The contents +% of this module will be \Tcllogo\ code that causes some embedded +% file to be stripped, sourced, and indexed. +% The catalogue code will be evaluated in a separate safe interpreter, +% so that a somewhat controlled set of commands can be made available. +% +% \begin{variable}{thefile} +% \begin{variable}{filename} +% The variables |thefile| and |filename| are used by +% \textsf{docstrip} catalogue commands as sources of information +% about the current file. |thefile| is the actual file contents, +% whereas |filename| is the name of the file (including a path, if +% one is needed). +% \end{variable}\end{variable} +% +% \begin{variable}{fileoptions} +% This variable holds the list of |fconfigure|-options for +% configuring a file before reading it. This information must be +% remembered, because it needs to be recorded in generated +% |package ifneeded| scripts. +% \end{variable} +% +% \begin{proc}{fileoptions} +% This command may be used in \textsf{docstrip} directories to +% change the set of options for files. The call syntax is +% \begin{quote} +% |fileoptions| \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% and the current set of options is set to precisely those +% specified (old options are forgotten). There is no particular +% return value, but the |thefile| variable contents are updated to +% reflect the new |fileoptions|. +% +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::fileoptions {args} { + variable filename + variable thefile [eval [list thefile $filename] $args] + variable fileoptions $args +} +%</utilpkg> +% \end{tcl} +% \end{proc} +% +% \begin{macrocode} +%<*utilman> +[section {Package indexing commands}] + +Like raw [file .tcl] files, code lines in docstrip source files can +be searched for package declarations and corresponding indices +constructed. A complication is however that one cannot tell from the +code blocks themselves which will fit together to make a working +package; normally that information would be found in an accompanying +[file .ins] file, but parsing one of those is not an easy task. +Therefore [package docstrip::util] introduces an alternative encoding +of such information, in the form of a declarative Tcl script: the +[term catalogue] (of the contents in a source file). +[para] + +The special commands which are available inside a catalogue are: +[list_begin definitions] +[call [cmd pkgProvide] [arg name] [arg version] [arg terminals]] + Declares that the code for a package with name [arg name] and + version [arg version] is made up from those modules in the source + file which are selected by the [arg terminals] list of guard + expression terminals. This code should preferably not contain a + [cmd {package}] [method {provide}] command for the package, as one + will be provided by the package loading mechanisms. +[call [cmd pkgIndex] [opt "[arg terminal] ..."]] + Declares that the code for a package is made up from those modules + in the source file which are selected by the listed guard + expression [arg terminal]s. The name and version of this package is + determined from [cmd {package}] [method {provide}] command(s) found + in that code (hence there must be such a command in there). +[call [cmd fileoptions] [opt "[arg option] [arg value] ..."]] + Declares the [cmd fconfigure] options that should be in force when + reading the source; this can usually be ignored for pure ASCII + files, but if the file needs to be interpreted according to some + other [option -encoding] then this is how to specify it. The + command should normally appear first in the catalogue, as it takes + effect only for commands following it. +[list_end] +Other Tcl commands are supported too [vset emdash] a catalogue is +parsed by being evaluated in a safe interpreter [vset emdash] but they +are rarely needed. To allow for future extensions, unknown commands +in the catalogue are silently ignored. +[para] + +To simplify distribution of catalogues together with their source +files, the catalogue is stored [emph {in the source file itself}] as +a module selected by the terminal '[const docstrip.tcl::catalogue]'. +This supports both the style of collecting all catalogue lines in one +place and the style of putting each catalogue line in close proximity +of the code that it declares. +[para] + +% \end{macrocode} +% \DontCheckModules +% \begin{Macrocode} +Putting catalogue entries next to the code they declare may look as +follows +[example { +%<<verbatim +% First there's the catalogue entry +% \begin{tcl} +%<docstrip.tcl::catalogue>pkgProvide foo::bar 1.0 {foobar load} +% \end{tcl} +% second a metacomment used to include a copyright message +% \begin{macrocode} +%<*foobar> +%% This file is placed in the public domain. +% \end{macrocode} +% third the package implementation +% \begin{tcl} +namespace eval foo::bar { + # ... some clever piece of Tcl code elided ... +% \end{tcl} +% which at some point may have variant code to make use of a +% |load|able extension +% \begin{tcl} +%<*load> + load [file rootname [info script]][info sharedlibextension] +%</load> +%<*!load> + # ... even more clever scripted counterpart of the extension + # also elided ... +%</!load> +} +%</foobar> +% \end{tcl} +% and that's it! +%verbatim +}] +The corresponding set-up with [cmd pkgIndex] would be +[example { +%<<verbatim +% First there's the catalogue entry +% \begin{tcl} +%<docstrip.tcl::catalogue>pkgIndex foobar load +% \end{tcl} +% second a metacomment used to include a copyright message +% \begin{tcl} +%<*foobar> +%% This file is placed in the public domain. +% \end{tcl} +% third the package implementation +% \begin{tcl} +package provide foo::bar 1.0 +namespace eval foo::bar { + # ... some clever piece of Tcl code elided ... +% \end{tcl} +% which at some point may have variant code to make use of a +% |load|able extension +% \begin{tcl} +%<*load> + load [file rootname [info script]][info sharedlibextension] +%</load> +%<*!load> + # ... even more clever scripted counterpart of the extension + # also elided ... +%</!load> +} +%</foobar> +% \end{tcl} +% and that's it! +%verbatim +% \end{Macrocode} +% \CheckModules +% \begin{macrocode} +}] +%</utilman> +% \end{macrocode} +% +% \begin{variable}{Report} +% Since commands in the catalogue will often be implemented as +% doing something, there is a need for giving them a way of +% reporting back what they did, as the basic ``report in return +% value'' idiom doesn't work for scripts. Hence there is a variable +% |Report| where information can be gathered. The value of this +% list is a list to which new items can be appended, although in +% the end they will typically be |join|ed with a newline as +% separator (thus blurring the distinction between a multiline item +% and multiple one-line items). +% \end{variable} +% +% \begin{proc}{Report} +% Normally, items are contributed to the report using the call +% \begin{quote} +% |Report| \word{item} +% \end{quote} +% where the \word{item} is some human-readable string. +% \begin{variable}{Report_store} +% \begin{variable}{Report_cmd} +% A complication is that one sometimes wants reports to be +% returned by the top level command, but other times one wants +% them to be written to the controlling terminal immediately +% (e.g. to give feedback of progress). The |Report| mechanism +% aims to support both by having the action of the |Report| +% command controlled by one boolean variables |Report_store| and +% one command prefix |Report_cmd|. If the former is true, then +% the \word{item} is appended to the |Report| list. Moreover +% the latter is evaluated with the \word{item} as an extra +% argument. To have the latter ``do nothing'', use |list|. +% \end{variable}\end{variable} +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::Report {item} { + variable Report_store + if {$Report_store} then { + variable Report + lappend Report $item + } + variable Report_cmd + eval [linsert $Report_cmd end $item] +} +%</utilpkg> +% \end{tcl} +% \end{proc} +% +% +% \subsection{Index entry generation} +% +% \begin{proc}{index_from_catalogue} +% The |index_from_catalogue| command generates package index data +% by reading catalogue modules in master source files and appends +% these entries to the relevant \texttt{pkgIndex.tcl} file. The +% call syntax is +% \changes{1.3}{2010/04/20}{Promoted the directory to being +% a mandatory argument, for symmetry with +% \texttt{pkg\PrintChar{95}mkIndex}. (LH)} +% \begin{quote} +% |docstrip::util::index_from_catalogue| \word{directory} +% \word{pattern} \begin{regblock}[\regstar]\word{option} +% \word{value}\end{regblock} +% \end{quote} +% where \word{directory} is the directory whose +% \texttt{pkgIndex.tcl} should be updated and \word{pattern} is a +% |glob|-pattern for files to read catalogues in. Currently the +% following \word{option}s are implemented: +% \begin{ttdescription} +% \item[-recursein] +% If nonempty, then the operation will be repeated in each +% subdirectory matching the pattern specified as \word{value}. +% |-recursein *| causes the entire subtree rooted at the +% \word{directory} to be processed. +% \item[-options] +% \textsf{Docstrip} expressions terminals in addition to +% the basic \texttt{docstrip.tcl::catalogue} to use when +% extracting the catalogue; a sort of meta-configuration +% facility. +% \item[-sourceconf] +% |fconfigure| options applied to the source file, before +% reading. |fileoptions| commands in the catalogue will +% override this setting (completely replacing the set of +% options); this will primarily control what is used when +% extracting the catalogue. Defaults to empty. +% \item[-report] +% Takes a boolean value. If true, the report will be the return +% value of |index_from_catalogue|. Defaults to false, in which +% case there is no particular return value. +% \item[-reportcmd] +% Takes a command prefix as value, which will be called as +% \begin{quote} +% \meta{prefix} \word{item} +% \end{quote} +% for every \word{item} being reported. Defaults to +% |puts stdout|; use |list| to effectively disable this feature. +% The return value from the prefix is ignored. +% \item[-RecursionDepth] +% An internal option used when making a recursive call to +% signal the distance to the top invokation. A positive value +% means ``don't bother about initialising the |Report| system.'' +% \end{ttdescription} +% +% \begin{macrocode} +%<*utilman> +[list_begin definitions] +[call [cmd docstrip::util::index_from_catalogue] [arg dir]\ + [arg pattern] [opt "[arg option] [arg value] ..."]] + This command is a sibling of the standard [cmd pkg_mkIndex] + command, in that it adds package entries to [file pkgIndex.tcl] + files. The difference is that it indexes [syscmd docstrip]-style + source files rather than raw [file .tcl] or loadable library files. + Only packages listed in the catalogue of a file are considered. + [para] + + The [arg dir] argument is the directory in which to look for files + (and whose [file pkgIndex.tcl] file should be amended). + The [arg pattern] argument is a [cmd glob] pattern of files to look + into; a typical value would be [const *.dtx] or + [const *.{dtx,ddt}]. Remaining arguments are option-value pairs, + where the supported options are: + [list_begin options] + [opt_def -recursein [arg dirpattern]] + If this option is given, then the [cmd index_from_catalogue] + operation will be repeated in each subdirectory whose name + matches the [arg dirpattern]. [option -recursein] [const *] will + cause the entire subtree rooted at [arg dir] to be indexed. + [opt_def -sourceconf [arg dictionary]] + Specify [cmd fileoptions] to use when reading the catalogues of + files (and also for reading the packages if the catalogue does + not contain a [cmd fileoptions] command). Defaults to being + empty. Primarily useful if your system encoding is very different + from that of the source file (e.g., one is a two-byte encoding + and the other is a one-byte encoding). [const ascii] and + [const utf-8] are not very different in that sense. + [opt_def -options [arg terminals]] + The [arg terminals] is a list of terminals in addition to + [const docstrip.tcl::catalogue] that should be held as true when + extracting the catalogue. Defaults to being empty. This makes it + possible to make use of "variant sections" in the catalogue + itself, e.g. gaurd some entries with an extra "experimental" and + thus prevent them from appearing in the index unless that is + generated with "experimental" among the [option -options]. + [opt_def -report [arg boolean]] + If the [arg boolean] is true then the return value will be a + textual, probably multiline, report on what was done. Defaults + to false, in which case there is no particular return value. + [opt_def -reportcmd [arg commandPrefix]] + Every item in the report is handed as an extra argument to the + command prefix. Since [cmd index_from_catalogue] would typically + be used at a rather high level in installation scripts and the + like, the [arg commandPrefix] defaults to + "[cmd puts] [const stdout]". + Use [cmd list] to effectively disable this feature. The return + values from the prefix are ignored. + [list_end] + + The [cmd {package ifneeded}] scripts that are generated contain + one [cmd {package require docstrip}] command and one + [cmd docstrip::sourcefrom] command. If the catalogue entry was + of the [cmd pkgProvide] kind then the [cmd {package ifneeded}] + script also contains the [cmd {package provide}] command. + [para] + + Note that [cmd index_from_catalogue] never removes anything from an + existing [file pkgIndex.tcl] file. Hence you may need to delete it + (or have [cmd pkg_mkIndex] recreate it from scratch) before running + [cmd index_from_catalogue] to update some piece of information, such + as a package version number. + [para] +%</utilman> +% \end{macrocode} +% +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::index_from_catalogue {dir pattern args} { + array set O { + -options "" + -sourceconf "" + -report 0 + -reportcmd {puts stdout} + -RecursionDepth 0 + } + array set O $args + if {$O(-RecursionDepth)==0} then { + variable Report {} Report_store $O(-report) \ + Report_cmd $O(-reportcmd) + } +% \end{tcl} +% The first step is to make sure that there is a +% \texttt{pkgIndex.tcl} file to append to. +% \begin{tcl} + set targetFn [file join $dir pkgIndex.tcl] + Report "Entries will go to: $targetFn" + if {![file exists $targetFn]} then { + Report "Generating empty index file." + set F [open $targetFn w] + puts $F {# Tcl package index file, version 1.1} + puts $F {# This file is generated by the "pkg_mkIndex" command} + puts $F {# and sourced either when an application starts up or} + puts $F {# by a "package unknown" script. It invokes the} + puts $F {# "package ifneeded" command to set up package-related} + puts $F {# information so that packages will be loaded automatically} + puts $F {# in response to "package require" commands. When this} + puts $F {# script is sourced, the variable $dir must contain the} + puts $F {# full path name of this file's directory.} + close $F + } +% \end{tcl} +% The second step is to gather the |package ifneeded| scripts for +% the directory in question. This involves creating a temporary helper +% interpreter for parsing the \Module{docstrip.tcl::catalogue}. +% \begin{tcl} + set c [interp create -safe] + $c eval { + proc unknown args {} + } + $c alias pkgProvide [namespace which PkgProvide] + $c alias pkgIndex [namespace which PkgIndex] + $c alias fileoptions [namespace which fileoptions] + variable PkgIndex "" + foreach fn [glob -nocomplain -directory $dir -tails $pattern] { + Report "Processing file: $fn" + variable filename [file join $dir $fn] + variable fileoptions $O(-sourceconf) + variable thefile [eval [list thefile $filename] $fileoptions] + set catalogue [extract $thefile\ + [linsert $O(-options) 0 docstrip.tcl::catalogue]\ + -metaprefix {#} -onerror puts] + $c eval $catalogue + } + interp delete $c +% \end{tcl} +% The third step is easy: append the gathered material to the file. +% A header is inserted that records the |-options| and +% |-sourceconf| settings that were used. +% \begin{tcl} + if {$PkgIndex ne ""} then { + set F [open $targetFn {WRONLY APPEND}] + set cmd [list docstrip::util::index_from_catalogue $dir $pattern] + if {$O(-options) ne ""} then { + lappend cmd -options $O(-options) + } + if {$O(-sourceconf) ne ""} then { + lappend cmd -sourceconf $O(-sourceconf) + } + puts $F "\n## Appendix generated by:\n## $cmd$PkgIndex" + close $F + } +% \end{tcl} +% Finally, the procedure may recurse into subdirectories and do the +% same things there. +% \begin{tcl} + if {[info exists O(-recursein)]} then { + incr O(-RecursionDepth) + foreach fn [ + glob -nocomplain -tails -types d -directory $dir\ + $O(-recursein) + ] { + eval [list index_from_catalogue [file join $dir $fn] $pattern]\ + [array get O] + } + } + if {$O(-RecursionDepth)==0 && $O(-report)} then { + return [join $Report \n] + } +} +% \end{tcl} +% \end{proc} +% +% \begin{variable}{PkgIndex} +% The |PkgIndex| variable stores material that should be written +% to the \texttt{pkgIndex.tcl} file. The |PkgIndex| procedure +% appends suitable |package ifneeded| commands to it. Each command +% must have a newline in front of it. +% \end{variable} +% +% \begin{proc}{PkgProvide} +% The |PkgProvide| procedure is an implementation of the |pkgProvide| +% command in \textsf{docstrip} directories. It generates +% |package ifneeded| commands and appends them to the |PkgIndex| +% variable. The call syntax is +% \begin{quote} +% |pkgProvide| \word{pkg-name} \word{version} \word{terminal-list} +% \end{quote} +% where the \word{terminal}s are the true terminals in guard +% expressions. There is no particular return value. +% +% The |package ifneeded| scripts generated have the form +% \begin{quote} +% |package provide |\word{pkg-name} \word{version}\\ +% |package require docstrip|\\ +% |docstrip::sourcefrom |\word{filename} \word{terminal-list} +% \meta{fileoptions} +% \end{quote} +% (except that semicolons rather than newlines are used as command +% separators). That the |package provide| command gets embedded in +% the script like may seem unintuitive, but the same thing is done +% in the |package ifneeded| scripts generated for \texttt{.tm} +% files. Also note that the \word{filename} must be constructed +% when the index file is |source|d; this mix of static and dynamic +% data leads to a certain amount of Quoting Hell. +% +% First, better check that the \word{version} is valid. +% \begin{tcl} +proc docstrip::util::PkgProvide {pkg ver terminals} { + if {[catch {package vcompare 0 $ver}]} then { + Report "Malformed version number $ver given for package $pkg." + return + } + variable PkgIndex + variable filename + variable fileoptions +% \end{tcl} +% Since command substitution will have to happen inside the script +% argument of |package ifneeded|, that word is quote-delimited. The +% previous words are straightforwardly handled by |list|-quoting. +% \begin{tcl} + append PkgIndex \n [list package ifneeded $pkg $ver] { "} +% \end{tcl} +% The |package provide| command is fixed and can thus be handled by +% a |list|-quoting here and now, but since that |list|-quoted +% string is then embedded into a quote-delimited word, any +% characters in it that trigger substitutions or terminate the word +% must be escaped. However, there will only be such characters +% around for very bizarre choices of package name. +% \begin{tcl} + append PkgIndex [string map {\\ {\\} \$ {\$} \[ {\[} \" {\"}}\ + [list package provide $pkg $ver]] {; } +% \end{tcl} +% The |package require docstrip| command is at least harmless. +% \begin{tcl} + append PkgIndex {package require docstrip} {; } +% \end{tcl} +% But for the |docstrip::sourcefrom| command, a different technique +% is used. Here, there will be a quoting |list| command present in +% the \texttt{pkgIndex.tcl} file, to be evaluated when that is +% |source|d, and therefore the quote-delimited nature of the +% enclosing word becomes irrelevant; a simple |list|-quoting of +% data to be embedded as command arguments is again sufficient. +% \begin{tcl} + append PkgIndex {[list docstrip::sourcefrom }\ + {[file join $dir } [list [file tail $filename]] {] }\ + [linsert $fileoptions 0 $terminals] {]"} +} +% \end{tcl} +% \end{proc} +% +% \begin{proc}{PkgIndex} +% The |PkgIndex| procedure is an implementation of the |pkgIndex| +% command in \textsf{docstrip} directories. It generates +% |package ifneeded| commands and appends them to the |PkgIndex| +% variable. The call syntax is +% \begin{quote} +% |pkgIndex| \word{terminal}\regstar +% \end{quote} +% where the \word{terminal}s are the true terminals in guard +% expressions. There is no particular return value. +% +% \begin{tcl} +proc docstrip::util::PkgIndex {args} { + variable thefile + if {[catch { + packages_provided [extract $thefile $args -metaprefix {#}] + } res]} then { + if {[lindex $::errorCode 0] eq "DOCSTRIP"} then { + Report "Stripping error \"$res\"\nwhile indexing module\ + <[join $args ,]>." + } else { + Report "Code evaluation error:\n $res\nwhile indexing\ + module <[join $args ,]>." + } + } else { + variable filename + variable PkgIndex + variable fileoptions + foreach {pkg ver} $res { + append PkgIndex \n [list package ifneeded $pkg $ver] { "} + append PkgIndex {package require docstrip} {; } + append PkgIndex {[list docstrip::sourcefrom }\ + {[file join $dir } [list [file tail $filename]] {] }\ + [linsert $fileoptions 0 $args] {]"} + } + } +} +%</utilpkg> +% \end{tcl} +% \end{proc} +% +% +% +% \subsection{Module generation} +% +% An alternative to package indices is to create \Tcllogo\ +% Module~(\texttt{.tm}) files. +% +% \begin{proc}{modules_from_catalogue} +% This procedure scans the \Module{docstrip.tcl::catalogue} of a +% \texttt{.dtx} file and writes out \Tcllogo\ module files for the +% packages it finds. The call syntax is +% \begin{quote} +% |modules_from_catalogue| \word{target root} \word{source file} +% \begin{regblock}[\regstar] \word{option} \word{value} +% \end{regblock} +% \end{quote} +% where the \word{target root} is the directory used as starting +% point for the paths builts from package names are generated, and +% \word{source file} is the file to process. The supported +% \word{option}s are: +% \begin{ttdescription} +% \item[-formatpostamble] +% Command prefix used to format postamble messages. The call +% syntax is +% \begin{quote} +% \meta{prefix} \word{message} \word{target filename} +% \word{source filename} \word{terminal-list} +% \end{quote} +% and the return value is the formatted message. Defaults to +% |classical_postamble {##}|. +% \item[-formatpreamble] +% Command prefix used to format preamble messages. The call +% syntax is +% \begin{quote} +% \meta{prefix} \word{message} \word{target filename} +% \word{source filename} \word{terminal-list} +% \end{quote} +% and the return value is the formatted message. Defaults to +% |classical_preamble {##}|. +% \item[-options] +% \textsf{Docstrip} expressions terminals in addition to +% the basic \texttt{docstrip.tcl::catalogue} to use when +% extracting the catalogue. A sort of meta-configuration +% facility. +% \item[-postamble] +% Message to put at the top of the generated file. Defaults to +% being empty. See also |-formatpostamble|. +% \item[-preamble] +% Message to put at the top of the generated file. Defaults to +% a space (which ends up contributing an empty line). See also +% |-formatpreamble|. +% \item[-report] +% Takes a boolean value. If true, the report will be the return +% value of |modules_from_catalogue|. If false, there is no +% particular return value. The default is true. +% \item[-reportcmd] +% Takes a command prefix as value, which will be called as +% \begin{quote} +% \meta{prefix} \word{item} +% \end{quote} +% for every \word{item} being reported. Defaults to |list|, +% which effectively disables this feature. +% The return value from the prefix is ignored. +% \item[-sourceconf] +% |fconfigure| options applied to the source file, before +% reading. +% \end{ttdescription} +% +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::modules_from_catalogue] [arg target]\ + [arg source] [opt "[arg option] [arg value] ..."]] + This command is an alternative to [cmd index_from_catalogue] which + creates Tcl Module ([file .tm]) files rather than + [file pkgIndex.tcl] entries. Since this action is more similar to + what [syscmd docstrip] classically does, it has features for + putting pre- and postambles on the generated files. + [para] + + The [arg source] argument is the name of the source file to + generate [file .tm] files from. The [arg target] argument is the + directory which should count as a module path, i.e., this is what + the relative paths derived from package names are joined to. The + supported options are: + [list_begin options] + [opt_def -preamble [arg message]] + A message to put in the preamble (initial block of comments) of + generated files. Defaults to a space. May be several lines, which + are then separated by newlines. Traditionally used for copyright + notices or the like, but metacomment lines provide an alternative + to that. + [opt_def -postamble [arg message]] + Like [option -preamble], but the message is put at the end of the + file instead of the beginning. Defaults to being empty. + [opt_def -sourceconf [arg dictionary]] + Specify [cmd fileoptions] to use when reading the catalogue of + the [arg source] (and also for reading the packages if the + catalogue does not contain a [cmd fileoptions] command). Defaults + to being empty. Primarily useful if your system encoding is very + different from that of the source file (e.g., one is a two-byte + encoding and the other is a one-byte encoding). [const ascii] and + [const utf-8] are not very different in that sense. + [opt_def -options [arg terminals]] + The [arg terminals] is a list of terminals in addition to + [const docstrip.tcl::catalogue] that should be held as true when + extracting the catalogue. Defaults to being empty. This makes it + possible to make use of "variant sections" in the catalogue + itself, e.g. gaurd some entries with an extra "experimental" guard + and thus prevent them from contributing packages unless those are + generated with "experimental" among the [option -options]. + [opt_def -formatpreamble [arg commandPrefix]] + Command prefix used to actually format the preamble. Takes four + additional arguments [arg message], [arg targetFilename], + [arg sourceFilename], and [arg terminalList] and returns a fully + formatted preamble. Defaults to using [cmd classical_preamble] + with a [arg metaprefix] of '##'. + [opt_def -formatpostamble [arg commandPrefix]] + Command prefix used to actually format the postamble. Takes four + additional arguments [arg message], [arg targetFilename], + [arg sourceFilename], and [arg terminalList] and returns a fully + formatted postamble. Defaults to using [cmd classical_postamble] + with a [arg metaprefix] of '##'. + [opt_def -report [arg boolean]] + If the [arg boolean] is true (which is the default) then the return + value will be a textual, probably multiline, report on what was + done. If it is false then there is no particular return value. + [opt_def -reportcmd [arg commandPrefix]] + Every item in the report is handed as an extra argument to this + command prefix. Defaults to [cmd list], which effectively disables + this feature. The return values from the prefix are ignored. Use + for example "[cmd puts] [const stdout]" to get report items + written immediately to the terminal. + [list_end] + An existing file of the same name as one to be created will be + overwritten. +%</utilman> +% \end{macrocode} +% +% Most of the actual work is done by the |GenerateNamedPkg| +% and\slash or |GeneratePkg| procedures. +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::modules_from_catalogue {target source args} { + array set Opt { + -formatpostamble {classical_postamble {##}} + -formatpreamble {classical_preamble {##}} + -options {} + -postamble {} + -preamble { } + -sourceconf {} + -report 1 + -reportcmd list + } + array set Opt $args + variable filename $source + variable fileoptions $Opt(-sourceconf) + variable thefile [eval [list thefile $source] $fileoptions] + variable Report {} Report_store $Opt(-report) \ + Report_cmd $Opt(-reportcmd) + set catalogue [extract $thefile\ + [linsert $Opt(-options) 0 docstrip.tcl::catalogue]\ + -metaprefix {#} -onerror puts] + set c [interp create -safe] + $c eval { + proc unknown args {} + } + $c alias pkgProvide\ + [namespace which GenerateNamedPkg] $target\ + [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\ + [linsert $Opt(-formatpostamble) end $Opt(-postamble)] + $c alias pkgIndex\ + [namespace which GeneratePkg] $target\ + [linsert $Opt(-formatpreamble) end $Opt(-preamble)]\ + [linsert $Opt(-formatpostamble) end $Opt(-postamble)] + $c alias fileoptions [namespace which fileoptions] + $c eval $catalogue + interp delete $c + if {$Opt(-report)} then {return [join $Report \n]} +} +% \end{tcl} +% \end{proc} +% +% \begin{proc}{GenerateNamedPkg} +% This procedure is an implementation of the |pkgProvide| catalogue +% command. The call syntax is +% \begin{quote} +% |GenerateNamedPkg| \word{target} \word{preamble-prefix} +% \word{postamble-prefix} \word{pkg-name} \word{version} +% \word{terminal-list} +% \end{quote} +% i.e., the alias should provide the first three arguments. +% \word{target} is the same as the \word{target} argument of +% |modules_from_catalogue|. The \word{preamble-prefix} and +% \word{postamble-prefix} arguments are command prefixes with the +% syntax +% \begin{quote} +% \meta{prefix} \word{target filename} \word{source filename} +% \word{terminal-list} +% \end{quote} +% which will return the preamble and postamble texts respectively +% for the generated module file. +% +% The first part is extracting and handling extraction errors. +% \begin{tcl} +proc docstrip::util::GenerateNamedPkg\ + {target preamblecmd postamblecmd name version terminals} { + variable thefile + if {[catch { + extract $thefile $terminals -metaprefix {#} + } text]} then { + Report "Stripping error \"$text\"\nwhile indexing module\ + <[join $terminals ,]>." + } else { +% \end{tcl} +% but after that it's all about generating the \texttt{.tm} file. +% Mapping |::| directly to |/| is a bit coarse, but it is what +% |::tcl::tm::UnknownHandler| does. Trimming away extra slashes +% protects against someone picking a package name beginning with +% |::|. +% \begin{tcl} + variable filename + set module [format {%s-%s.tm}\ + [string trim [string map {:: /} $name] /] $version] + set modL [file split $module] + file mkdir [file join $target [file dirname $module]] + set F [open [file join $target $module] w] + fconfigure $F -encoding utf-8 + puts $F [eval $preamblecmd [list $module $filename $terminals]] + puts -nonewline $F $text + puts $F [eval $postamblecmd [list $module $filename $terminals]] + close $F + Report "Wrote $module" + } +} +% \end{tcl} +% \end{proc} +% +% +% \begin{proc}{GeneratePkg} +% This procedure is an implementation of the |pkgIndex| catalogue +% command. It is basically the same as |GenerateNamedPkg|, but it +% must also (i)~index the extracted code to find out the package +% name and version, and (ii)~handle the case that the code declares +% several packages, by generating redirection files. +% The call syntax is +% \begin{quote} +% |GeneratePkg| \word{target} \word{preamble-prefix} +% \word{postamble-prefix} \word{terminal}\regstar +% \end{quote} +% i.e., the alias should provide the first three arguments. +% \word{target} is the same as the \word{target} argument of +% |modules_from_catalogue|. The \word{preamble-prefix} and +% \word{postamble-prefix} arguments are command prefixes with the +% syntax +% \begin{quote} +% \meta{prefix} \word{target filename} \word{source filename} +% \word{terminal-list} +% \end{quote} +% which will return the preamble and postamble texts respectively +% for the generated module file. +% +% The first part is extracting and looking for package +% declarations, including the handling of errors during these +% operations. +% \begin{tcl} +proc docstrip::util::GeneratePkg {target preamblecmd postamblecmd args} { + variable thefile + if {[catch { + set text [extract $thefile $args -metaprefix {#}] + packages_provided $text + } res]} then { + if {[lindex $::errorCode 0] eq "DOCSTRIP"} then { + Report "Stripping error \"$res\"\nwhile indexing module\ + <[join $args ,]>." + } else { + Report "Code evaluation error:\n $res\nwhile indexing\ + module <[join $args ,]>." + } +% \end{tcl} +% There's also the corner case of not fining any package +% declaration, +% \begin{tcl} + } elseif {![llength $res]} then { + Report "Found no package in module <[join $args ,]>." + } else { +% \end{tcl} +% but after that it's all about generating \texttt{.tm} files. +% Mapping |::| directly to |/| is a bit coarse, but it is what +% |::tcl::tm::UnknownHandler| does. Trimming away extra slashes +% protects against someone picking a package name beginning with +% |::|. +% \begin{tcl} + variable filename + set module [format {%s-%s.tm}\ + [string trim [string map {:: /} [lindex $res 0]] /]\ + [lindex $res 1]] + set modL [file split $module] + file mkdir [file join $target [file dirname $module]] + set F [open [file join $target $module] w] + fconfigure $F -encoding utf-8 + puts $F [eval $preamblecmd [list $module $filename $args]] + puts -nonewline $F $text + puts $F [eval $postamblecmd [list $module $filename $args]] + close $F + Report "Wrote $module" +% \end{tcl} +% Now, it might happen that a module provides more than package, +% and what should then be done for the extra packages? A reasonable +% solution seems to be to generate \texttt{.tm} files for all of +% them, but make the extra files consist of a single |source| +% command for the first file. Starting from the runtime +% |info script| value it is possible to compute the expected location +% of that file, but constructing the code to do it is a bit of work. +% \begin{tcl} + foreach {pkg ver} [lreplace $res 0 1] { + set mod2 [format {%s-%s.tm}\ + [string trim [string map {:: /} $pkg] /] $ver] + set mod2L [file split $mod2] + file mkdir [file join $target [file dirname $mod2]] + set common 0 + foreach d1 $modL d2 $mod2L { + if {$d1 eq $d2} then {incr common} else {break} + } + set tail [lrange $modL $common end] + set script {[::info script]} + foreach d2 $mod2L { + if {[incr common -1] < 0} then { + set script "\[::file dirname $script\]" + } + } + set F [open [file join $target $mod2] w] + fconfigure $F -encoding utf-8 + puts $F "::source -encoding utf-8 \[::file join $script $tail\]" + close $F + Report "Wrote redirect $mod2" + } + } +} +% \end{tcl} +% \end{proc} +% +% \begin{proc}{classical_preamble} +% This procedure generates preambles in the style of the \LaTeX\ +% \textsf{docstrip} utility. It has the call syntax +% \begin{quote} +% |docstrip::util::classical_preamble| \word{metaprefix} +% \word{message} \word{target filename} +% \begin{regblock}[\regstar] \word{source filename} +% \word{terminal-list} \end{regblock} +% \end{quote} +% and returns the generated preamble. +% +% In comparison with \textsf{docstrip}, the \word{target filename} +% is |\outFileName|, the pairs of \word{source filename} and +% \word{terminal-list} are going to contribute to +% |\ReferenceLines|, and \word{message} is what gets added at the +% end. +% \begin{tcl} +proc docstrip::util::classical_preamble {metaprefix message target args} { + set res {""} + lappend res " This is `$target'," + lappend res { generated by the docstrip::util package.} + lappend res {} { The original source files were:} {} + foreach {source terminals} $args { + set line " [file tail $source]" + if {[llength $terminals]} then { + append line { (with options: `} [join $terminals ,] {')} + } + lappend res $line + } + foreach line [split $message \n] {lappend res " $line"} + return $metaprefix[join $res "\n$metaprefix"] +} +%</utilpkg> +% \end{tcl} +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::classical_preamble] [arg metaprefix]\ + [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]] + This command returns a preamble in the classical + [syscmd docstrip] style +[example { +## +## This is `TARGET', +## generated by the docstrip::util package. +## +## The original source files were: +## +## SOURCE (with options: `foo,bar') +## +## Some message line 1 +## line2 +## line3 +}] + if called as +[example_begin] +docstrip::util::classical_preamble {##}\ + "\nSome message line 1\nline2\nline3" TARGET SOURCE {foo bar} +[example_end] + The command supports preambles for files generated from multiple + sources, even though [cmd modules_from_catalogue] at present does + not need that. +%</utilman> +% \end{macrocode} +% \end{proc} +% +% \begin{proc}{classical_postamble} +% This procedure generates postambles in the style of the \LaTeX\ +% \textsf{docstrip} utility. It has the call syntax +% \begin{quote} +% |docstrip::util::classical_postamble| \word{metaprefix} +% \word{message} \word{target filename} +% \begin{regblock}[\regstar] \word{source filename} +% \word{terminal-list} \end{regblock} +% \end{quote} +% and returns the generated postamble. +% +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::classical_postamble {metaprefix message target args} { + set res {} + foreach line [split $message \n] {lappend res " $line"} + lappend res {} " End of file `$target'." + return $metaprefix[join $res "\n$metaprefix"] +} +%</utilpkg> +% \end{tcl} +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::classical_postamble] [arg metaprefix]\ + [arg message] [arg target] [opt "[arg source] [arg terminals] ..."]] + This command returns a postamble in the classical + [syscmd docstrip] style +[example { +## Some message line 1 +## line2 +## line3 +## +## End of file `TARGET'. +}] + if called as +[example_begin] +docstrip::util::classical_postamble {##}\ + "Some message line 1\nline2\nline3" TARGET SOURCE {foo bar} +[example_end] + In other words, the [arg source] and [arg terminals] arguments are + ignored, but supported for symmetry with [cmd classical_preamble]. +%</utilman> +% \end{macrocode} +% \end{proc} +% +% +% \subsection{Scanning for declarations} +% +% One task that must be performed is finding out which package(s) are +% provided by a particular script (which will typically constitute +% the contents of a module). +% +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::packages_provided] [arg text]\ + [opt [arg setup-script]]] + This command returns a list where every even index element is the + name of a package [cmd provide]d by [arg text] when that is + evaluated as a Tcl script, and the following odd index element is + the corresponding version. It is used to do package indexing of + extracted pieces of code, in the manner of [cmd pkg_mkIndex]. + [para] + + One difference to [cmd pkg_mkIndex] is that the [arg text] gets + evaluated in a safe interpreter. [cmd {package require}] commands + are silently ignored, as are unknown commands (which includes + [cmd source] and [cmd load]). Other errors cause + processing of the [arg text] to stop, in which case only those + package declarations that had been encountered before the error + will be included in the return value. + [para] + + The [arg setup-script] argument can be used to customise the + evaluation environment, if the code in [arg text] has some very + special needs. The [arg setup-script] is evaluated in the local + context of the [cmd packages_provided] procedure just before the + [arg text] is processed. At that time, the name of the slave + command for the safe interpreter that will do this processing is + kept in the local variable [var c]. To for example copy the + contents of the [var ::env] array to the safe interpreter, one + might use a [arg setup-script] of + [example { $c eval [list array set env [array get ::env]]}] +%</utilman> +% \end{macrocode} +% \begin{proc}{packages_provided} +% This procedure looks for package declarations inside a script. It +% is derived from |pkg_mkIndex|, but simplified as it does not load +% binary libraries or invoke autoloading. The call syntax is +% \begin{quote} +% |packages_provided| \word{text} \word{setup-script}\regopt +% \end{quote} +% where \word{text} is the text to scan. The result is a list +% \begin{quote} +% \begin{regblock}[\regstar]\word{package} +% \word{version}\end{regblock} +% \end{quote} +% of packages that were declared. +% +% The \word{setup-script} is meant as a hook useful when indexing +% packages that have some special need. This argument is a script +% that gets evaluated (in the |packages_provided| procedure) +% \emph{after} the test interpreter used for loading packages in +% has been set up, but \emph{before} the text to index is evaluated +% in it. The local |c| variable holds the name of the interpreter +% command. +% \changes{1.3}{2006/05/24}{Command added. (LH)} +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::packages_provided {text {setup ""}} { +% \end{tcl} +% First create the test interpreter and prepare it for use. Unlike +% the case in standard package indexing, this interpreter is safe +% (since safe interpreters are faster to create). Use the +% \word{setup-script} if you need to expose some unsafe feature. +% \changes{1.3}{2010/03/28}{Using safe interpreter for package +% indexing. (LH)} +% \begin{tcl} + set c [interp create -safe] + $c eval { + proc tclPkgUnknown args {} + package unknown tclPkgUnknown + proc unknown {args} {} + proc auto_import {args} {} + } + $c hide package + $c alias package [namespace which packages_provided,package] $c + eval $setup +% \end{tcl} +% Now evaluate the \word{text}. Errors are cheerfully ignored. Data +% for the return value is collected in the \describestring[local +% var.]{package_list}|package_list| local variable, which +% |packages_provided,package| uses |uplevel| to access. +% \begin{tcl} + set package_list {} + catch {$c eval $text} +% \end{tcl} +% Cleanup and return the result. +% \begin{tcl} + interp delete $c + return $package_list +} +% \end{tcl} +% \end{proc} +% +% \begin{proc}{packages_provided,package} +% Calls to |package| in the test interpreter will be routed through +% this procedure, so that |provide|s can be seen and |request|s can +% be ignored. This is different from the mechanism in +% |pkg_mkIndex|, but I think this approach is more correct. The +% call syntax is +% \begin{quote} +% |packages_provided,package| \word{interp} +% \word{subcommand} \word{argument}\regstar +% \end{quote} +% where \word{interp} is the slave interpreter command to use when +% actually carrying out the command. Remaining arguments are as for +% the core command |package|, which is assumed to be hidden in +% \word{interp}. +% \begin{tcl} +proc docstrip::util::packages_provided,package {interp subcmd args} { + switch -- $subcmd { + r - re - req - requ - requi - requir - require { + return + } + pro - prov - provi - provid - provide { + if {[llength $args] == 2} then { + uplevel 1 [list lappend package_list] $args + } + } + } + eval [list $interp invokehidden package $subcmd] $args +} +%</utilpkg> +% \end{tcl} +% \end{proc} +% +% \begin{macrocode} +%<*utilman> +[list_end] +%</utilman> +% \end{macrocode} +% +% +% \section{Operations on source file text} +% +% \begin{macrocode} +%<*utilman> + +[section {Source processing commands}] + +Unlike the previous group of commands, which would use +[cmd docstrip::extract] to extract some code lines and then process +those further, the following commands operate on text consisting of +all types of lines. + +[list_begin definitions] +%</utilman> +% \end{macrocode} +% +% +% +% \subsection{Supporting doctools as markup language} +% +% In the interest of making \textsf{docstrip} useful also for +% programmers who do not want to write \LaTeX\ markup, some support is +% offered also for files with \textsf{doctools} \texttt{.man} markup in +% the comment lines. It is suggested that such files are given the +% suffix \texttt{.ddt} to distinguish them from the \texttt{.dtx} files +% that are directly \LaTeX able. +% +% More precisely, it is suggested that the markup on comment and +% metacomment lines of a \texttt{.ddt} file should follow the syntax on +% the \texttt{doctools\_fmt} manpage~\cite{doctools_fmt}, or in the +% future perhaps some derivative thereof. Unlike the case in +% \texttt{.dtx} files, no explicit markup is required (or wanted) +% around blocks of code and guard lines; such markup is to be generated +% by the procedure below, as part of adding suitable markup to the code +% lines. +% +% \begin{proc}{ddt2man} +% This procedure takes a string in the \texttt{.ddt} format sketched +% above and returns the corresponding text with \textsf{doctools} +% \texttt{.man} markup. The syntax is +% \begin{quote} +% |docstrip::util::ddt2man| \word{text} +% \end{quote} +% +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::ddt2man] [arg text]] + The [cmd ddt2man] command reformats [arg text] from the general + [syscmd docstrip] format to [package doctools] [file .man] format + (Tcl Markup Language for Manpages). The different line types are + treated as follows: + [list_begin definitions] + [def {comment and metacomment lines}] + The '%' and '%%' prefixes are removed, the rest of the text is + kept as it is. + [def {empty lines}] + These are kept as they are. (Effectively this means that they will + count as comment lines after a comment line and as code lines + after a code line.) + [def {code lines}] + [cmd example_begin] and [cmd example_end] commands are placed + at the beginning and end of every block of consecutive code + lines. Brackets in a code line are converted to [cmd lb] and + [cmd rb] commands. + [def {verbatim guards}] + These are processed as usual, so they do not show up in the + result but every line in a verbatim block is treated as a code + line. + [def {other guards}] + These are treated as code lines, except that the actual guard is + [cmd emph]asised. + [list_end] + + At the time of writing, no project has employed [package doctools] + markup in master source files, so experience of what works well is + not available. A source file could however look as follows +[example { +%</utilman> +%<*utilman,gcdexample> +%<<verbatim +% [manpage_begin gcd n 1.0] +% [keywords divisor] +% [keywords math] +% [moddesc {Greatest Common Divisor}] +% [require gcd [opt 1.0]] +% [description] +% +% [list_begin definitions] +% [call [cmd gcd] [arg a] [arg b]] +% The [cmd gcd] procedure takes two arguments [arg a] and [arg b] which +% must be integers and returns their greatest common divisor. +proc gcd {a b} { +% The first step is to take the absolute values of the arguments. +% This relieves us of having to worry about how signs will be treated +% by the remainder operation. + set a [expr {abs($a)}] + set b [expr {abs($b)}] +% The next line does all of Euclid's algorithm! We can make do +% without a temporary variable, since $a is substituted before the +% [lb]set a $b[rb] and thus continues to hold a reference to the +% "old" value of [var a]. + while {$b>0} { set b [expr { $a % [set a $b] }] } +% In Tcl 8.3 we might want to use [cmd set] instead of [cmd return] +% to get the slight advantage of byte-compilation. +%<tcl83> set a +%<!tcl83> return $a +} +% [list_end] +% +% [manpage_end] +%verbatim +%</utilman,gcdexample> +%<*utilman> +}] + If the above text is fed through [cmd docstrip::util::ddt2man] then + the result will be a syntactically correct [package doctools] + manpage, even though its purpose is a bit different. + [para] + + It is suggested that master source code files with [package doctools] + markup are given the suffix [file .ddt], hence the "ddt" in + [cmd ddt2man]. + +%</utilman> +% \end{macrocode} +% +% The structure of this procedure is fairly similar to that of +% |extract|, although of course the processing of the lines is rather +% different. The main novelty is the variable |wascode|, which is +% true if the previous line was a code line of some sort. +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::ddt2man {text} { + set wascode 0 + set verbatim 0 + set res "" + foreach line [split $text \n] { + if {$verbatim} then { + if {$line eq $endverbline} then { + set verbatim 0 + } else { + append res [string map {[ [lb] ] [rb]} $line] \n + } + } else { + switch -glob -- $line %%* { + if {$wacode} then { + append res {[example_end]} \n + set wascode 0 + } + append res [string range $line 2 end] \n + } %<<* { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + set endverbline "%[string range $line 3 end]" + set verbatim 1 + } %<* { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + set guard "" + regexp -- {(^%<[^>]*>)(.*)$} $line "" guard line + append res \[ [list emph $guard] \]\ + [string map {[ [lb] ] [rb]} $line] \n + } %* { + if {$wascode} then { + append res {[example_end]} \n + set wascode 0 + } + append res [string range $line 1 end] \n + } {\\endinput} { + break + } "" { +% \end{tcl} +% Experience showed that empty lines at the beginning and end of a +% file were hard to avoid. In order to stop those from being marked +% up as examples, an empty line will not trigger a switch to code +% mode. +% \begin{tcl} + append res \n + } default { + if {!$wascode} then { + append res {[example_begin]} \n + set wascode 1 + } + append res [string map {[ [lb] ] [rb]} $line] \n + } + } + } + if {$wascode} then {append res {[example_end]} \n} + return $res +} +%</utilpkg> +% \end{tcl} +% There is no test of this procedure, since it is rather +% experimental. One could however develop the example above into a +% test, if the need seems significant. +% \end{proc} +% +% +% \subsection{Guard information} +% +% \begin{proc}{guards} +% The |guards| command looks through a piece of master source code +% and gathers information about the guards occurring therein. The +% syntax is +% \begin{quote} +% |docstrip::util::guards| \word{subcommand} \word{text} +% \end{quote} +% where the \word{subcommand} is one of the following: +% \begin{ttdescription} +% \item[names] +% Return the list of expression terminals occuring in the +% \word{text}, in no particular order. +% \item[counts] +% Return a dictionary which for each expression terminal +% occuring in the \word{text} gives the number of times it +% occurs. +% \item[expressions] +% Return the list of expressions occuring in the \word{text}, +% in no particular order. +% \item[exprcounts] +% Return a dictionary which for each guard expression occuring +% in the \word{text} gives the number of times it occurs. +% \item[exprmods] +% Return a dictionary which for each guard expression occuring +% in the \word{text} gives a string of the modifiers of these +% guards (where space is used for no modifier). This is the raw +% format of the information collected by this procedure. +% \item[exprerr] +% Return the list of syntactically incorrect expressions occuring +% in the \word{text}, in no particular order. +% \item[rotten] +% Return a dictionary which maps line numbers with bad guards to +% their contents. +% \end{ttdescription} +% \changes{1.1}{2005/03/02}{Command added. (LH, extending a +% suggestion of AK)} +% \changes{1.2}{2005/08/23}{Changed name of \texttt{badguards} +% subcommand to \texttt{rotten}. (LH)} +% \changes{1.2}{2005/08/26}{Changed name of \texttt{guard} +% procedure to \texttt{guards}. (LH)} +% \begin{tcl} +%<*utilman> +[call [cmd docstrip::util::guards] [arg subcmd] [arg text]] + The [cmd guards] command returns information (mostly of a + statistical nature) about the ordinary docstrip guards that occur + in the [arg text]. The [arg subcmd] selects what is returned. + + [list_begin definitions] + [def [method counts]] + List the guard expression terminals with counts. The format of + the return value is a dictionary which maps the terminal name to + the number of occurencies of it in the file. + [def [method exprcount]] + List the guard expressions with counts. The format of the return + value is a dictionary which maps the expression to the number of + occurencies of it in the file. + [def [method exprerr]] + List the syntactically incorrect guard expressions (e.g. + parentheses do not match, or a terminal is missing). The return + value is a list, with the elements in no particular order. + [def [method expressions]] + List the guard expressions. The return value is a list, with the + elements in no particular order. + [def [method exprmods]] + List the guard expressions with modifiers. The format of the return + value is a dictionary where each index is a guard expression and + each entry is a string with one character for every guard line that + has this expression. The characters in the entry specify what + modifier was used in that line: +, -, *, /, or (for guard without + modifier:) space. This is the most primitive form of the + information gathered by [cmd guards]. + [def [method names]] + List the guard expression terminals. The return value is a list, + with the elements in no particular order. + [def [method rotten]] + List the malformed guard lines (this does not include lines where + only the expression is malformed, though). The format of the return + value is a dictionary which maps line numbers to their contents. + [list_end] +%</utilman> +% \end{tcl} +% +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::guards {subcmd text} { +% \end{tcl} +% The first part is a cut-down |extract|. It collects data in the |E| +% array, which is indexed by expression. The |badL| variable is used +% for the data returned by the |rotten| subcommand. +% \begin{tcl} + set verbatim 0 + set lineno 1 + set badL {} + foreach line [split $text \n] { + if {$verbatim} then { + if {$line eq $endverbline} then {set verbatim 0} + } else { + switch -glob -- $line %<<* { + set endverbline "%[string range $line 3 end]" + set verbatim 1 + } %<* { + if {![ + regexp -- {^%<([*/+-]?)([^>]*)>(.*)$} $line ""\ + modifier expression line + ]} then { + lappend badL $lineno $line + } else { + if {$modifier eq ""} then {set modifier " "} + append E($expression) $modifier + } + } + } + incr lineno + } + if {$subcmd eq "rotten"} then {return $badL} +% \end{tcl} +% The second part processes the |E| array contents to produce the +% various subcommand results. +% \begin{tcl} + switch -- $subcmd "exprmods" { + return [array get E] + } "expressions" { + return [array names E] + } "exprerr" { + set res {} + foreach expr [array names E] { + regsub -all {[^()!,|&]+} $expr 0 e + regsub -all {,} $e {|} e + if {[catch {expr $e}]} then {lappend res $expr} + } + return $res + } + foreach name [array names E] { + set E($name) [string length $E($name)] + } + if {$subcmd eq "exprcounts"} then {return [array get E]} + foreach expr [array names E] { + foreach term [split $expr "()!,|&"] { + if {$term eq ""} then {continue} + if {![info exists T($term)]} then {set T($term) 0} + incr T($term) $E($expr) + } + } + switch -- $subcmd "counts" { + return [array get T] + } "names" { + return [array names T] + } default { + error "Unknown subcommand '$subcmd', must be one of:\ + counts, exprcounts, expressions, exprmods, names, rotten" + } +} +%</utilpkg> +% \end{tcl} +% \end{proc} +% +% +% +% \subsection{Backporting assistance} +% +% It is (sadly) not entirely uncommon that the Literate Programmer finds +% him- or herself with generated files that have been modified, even if +% they carry prominent notices saying ``Don't do that! Change the +% \emph{source} instead!''. When such changes are to the worse there is +% little problem, because erasing them is just a matter of regenerating +% the files in question, but often enough they instead contain useful +% improvements of the code that one would like to keep. This requires +% porting them back into the master source file, which in theory may +% seem like a minor copy-and-paste task, but in practice often gets +% frustrating because of the amount of navigating between the sites of +% different changes that one must perform. +% +% Ordinarily such backporting would be handled using patch files, and +% that is what will be done also in this case, but the fact that the +% file in which the modifications were made does not look like the +% source file means traditional patching tools are not immediately +% useful. The procedures defined below provides for +% \textsc{docstrip}-aware patching. +% +% +% +% \begin{proc}{patch} +% The |patch| procedure applies a list of diff hunks to a +% \textsc{docstrip} style master source file. +% \changes{1.2}{2005/06/20}{Procedure added. (LH)} +% The syntax is +% \begin{quote} +% |docstrip::util::patch| \word{source var.} \word{terminals} +% \word{fromtext} \word{diff} +% \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock} +% \end{quote} +% The \word{source var.} is the name in the calling context +% of a variable which contains the list of lines in the source +% to patch; patching thus means modifying this list. \word{diff} is +% the difference hunks to apply, and the \word{fromtext} is the text +% that diff is meant to modify. \word{terminals} is the list of +% terminals one should use to |extract| \word{fromtext} (or a part +% thereof) from the source. The return value is a sort of annotated +% diff file, where each hunk carries a comment on how it was applied. +% Hunks with empty comments (usually meaning ``hunk applied in full, +% no problems were observed'') are omitted from this report. +% +% The \word{option} \word{value} pairs may be used to further control +% what happens. Currently the following options are interpreted: +% \begin{ttdescription} +% \item[-matching] +% How is the \word{diff} matched against the \word{fromtext}? +% (Hunks that don't match are ignored.) The default is |exact|, +% which means each line must match. The alternatives are |none| +% (in which case no check is made, i.e., line numbers are silently +% assumed to be correct), |nonspace| (only non-whitespace +% characters are compared), and |anyspace| (any sequence of +% whitespace characters compare as a single space). +% \item[-metaprefix] +% Same as for |docstrip::extract|. +% \item[-trimlines] +% Same as for |docstrip::extract|. +% \end{ttdescription} +% +% The \word{diff} is a list of ``parsed differences'', the format of +% which is explained in Subsection~\ref{Ssec:Tcldiff}. +% +% The way this procedure operates is that it first establishes a +% correspondence between lines in the source and lines in the +% \word{fromtext}. The first part of this correspondence is determined +% by the source and \word{terminals}, and is complicated but univocal. +% The second part of this correspondence is given by a matching of +% extracted lines to \word{fromtext} lines, and this is typically +% simple but not necessarily unique, which means the user need to be +% aware of the heuristic used: the lines of the \word{fromtext} are +% read in sequence, and whenever one matching the line after that in +% the extracted text with which the most recent correspondence is +% found, then these two are considered to correspond to each other. +% This should work well with the generated files one typically finds, +% which consist of long intervals of lines corresponding exactly to +% extracted texts, surrounded by some pre- and postambles. With files +% generated from several source files it may be necessary to add some +% metacomment line to disambigue the pieces, but that is often not a +% problem. +% +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::patch {sourcevar termL fromtext diff args} { + upvar 1 $sourcevar SL + array set O {-trimlines 1 -matching exact} + array set O $args +% \end{tcl} +% The first step is to construct the array |lift| that maps +% \word{fromtext} line numbers to source line numbers. This array +% actually contains a bit more than just the line numbers; the +% complete entry format is +% \begin{quote} +% \word{SL index} \word{source-prefix} \word{extract-prefix} +% \end{quote} +% where \word{SL index} is an index into the source \emph{list} of +% lines (thus starting at zero rather than one), \word{source-prefix} +% is the source line prefix (usually an empty string) that was removed +% as part of the extraction process, and \word{extract-prefix} is the +% prefix that was inserted as part of the extraction process (usually +% an empty string). +% +% In order to gather the above information, |extract| is run in the two +% lines of annotation format, which means the interpretation of an |EL| +% element depends heavily on its index modulo $3$. Setting the last +% element to a newline (it would otherwise had been an empty string, +% since |extract| places a newline after every line) is a sneaky way +% of preventing the |ptr| ``pointer'' into |EL| from going past the +% end of that list. +% \begin{tcl} + set cmd [list extract [join $SL \n] $termL -annotate 2] + foreach opt {-metaprefix -trimlines} { + if {[info exists O($opt)]} then {lappend cmd $opt $O($opt)} + } + set EL [split [eval $cmd] \n] + lset EL end \n + set ptr 0 + set lineno 1 + set FL [list {}] + foreach line [split $fromtext \n] { + lappend FL $line + if {$O(-trimlines)} then {set line [string trimright $line " "]} + if {$line eq [lindex $EL $ptr]} then { + set lift($lineno) [lindex $EL [incr ptr]] + lset lift($lineno) 0 [expr { [lindex $EL [incr ptr]] - 1 }] + incr ptr + } + incr lineno + } +% \end{tcl} +% The |FL| variable constructed above is a list of \word{fromtext} +% lines, with list index equal to line number. It is used below when +% matching the differences. +% +% If at this point the |lift| array is empty, then no patching can be +% done. An error is thrown which suggests that the user checks the +% input given. +% \begin{tcl} + if {![array size lift]} then { + return -code error "The extract did not match any part of the\ + fromtext. Check the list of terminals and the options" + } +% \end{tcl} +% +% The second step consists of extending the \word{diff} to a +% ``replace-list'', so that the hunk format becomes +% \begin{quote} +% \word{start1} \word{end1} \word{start2} \word{end2} +% \word{lines} \word{replaces} +% \end{quote} +% where the \word{replaces} is a list of lists on the form +% \begin{quote} +% \word{start1} \word{end1} \word{line}\regstar +% \end{quote} +% i.e., the same format as for arguments two and up of |lreplace|. (In +% particular, \(\mathit{end1} = \mathit{start1}-1\) when only +% inserting and there are no \word{line}s when only removing.) These +% extended hunks are placed in the variable |RL| sorted in descending +% order after \word{start1}, and the replaces within each hunk are +% sorted in that order too. +% +% This is also where the procedure begins constructing its report, +% which is another extension of the hunk format. Here the syntax is +% \begin{quote} +% \word{start1} \word{end1} \word{start2} \word{end2} +% \word{lines} \word{comment} +% \end{quote} +% where the \word{comment} is a description of what was done with +% this hunk. +% \begin{tcl} + set RL [list] + set log [list] + foreach hunk [lsort -decreasing -integer -index 0 $diff] { + set replL [list] + set l1 [lindex $hunk 0] + set repl {0 -1} + set matches 1 + foreach {type line} [lindex $hunk 4] { + switch -glob -- $type {[0-]} { + switch -- $O(-matching) "exact" { + if {[lindex $FL $l1] ne $line} then {set matches 0} + } "nonspace" { + if {[regsub -all -- {\s} $line {}] ne\ + [regsub -all -- {\s} [lindex $FL $l1] {}]} then { + set matches 0 + } + } "anyspace" { + if {[regsub -all -- {\s+} $line { }] ne\ + [regsub -all -- {\s+} [lindex $FL $l1] { }]} then { + set matches 0 + } + } + } + switch -- $type synch { + if {[llength $repl]>2 ||\ + [lindex $repl 1]-[lindex $repl 0]>=0} then { + lappend replL $repl + } + set repl [list $l1 [expr {$l1-1}]] + } + { + lappend repl $line + } - { + lset repl 1 $l1 + incr l1 + } 0 { + if {[llength $repl]>2 ||\ + [lindex $repl 1]-[lindex $repl 0]>=0} then { + lappend replL $repl + set repl {0 -1} + } + lset repl 1 $l1 + incr l1 + lset repl 0 $l1 + } + } + if {[llength $repl]>2 || [lindex $repl 1]-[lindex $repl 0]>=0}\ + then {lappend replL $repl} + if {$matches} then { + lappend hunk [lsort -decreasing -integer -index 0 $replL] + lappend RL $hunk + } else { + lappend hunk "(-- did not match fromtext --)" + lappend log $hunk + } + } +% \end{tcl} +% The difference granularity is now the one that will be used in the +% insertion of new lines. The reason for extending hunks rather than +% using something else is to use the original data when reporting +% problems. +% +% The third step is to actually apply the changes to |SL|, translating +% line numbers as one goes along. Differences are processed +% back-to-front, because that means first file line numbers are valid, +% and those are the ones that can be translated to source line numbers. +% +% \begin{tcl} + foreach hunk $RL { + set applied 0 + set misapplied 0 +% \end{tcl} +% For the purpose of generating a report, count is kept of how many +% lines of each hunk could be applied or could not be applied. That +% a hunk could not be applied (|applied| is zero) is often normal +% (the changed material was not generated from this source file), +% but if both counters are positive at the same time then one should +% take a bit more notice. +% \begin{tcl} + foreach repl [lindex $hunk 5] { + unset -nocomplain from to +% \end{tcl} +% A |repl| is processed by replacing the item range |$from|--|$to| +% of lines to remove by the lines to insert, but for that the entire +% range of source lines must be continuous. The |for| loop below +% sets |from| and |to| to the endpoints of the first range of lines +% to replace because of this |repl|, and removes any subsequent +% ranges immediately. +% \begin{tcl} + for {set n [lindex $repl 1]} {$n>=[lindex $repl 0]}\ + {incr n -1} { + if {![info exists lift($n)]} then { + incr misapplied + continue + } elseif {![info exists from]} then { + set to [lindex $lift($n) 0] + set from $to + } elseif {[lindex $lift($n) 0] == $from-1} then { + set from [lindex $lift($n) 0] + } else { + set SL [lreplace $SL $from $to] + set to [lindex $lift($n) 0] + set from $to + } + incr applied + set n0 $n + } +% \end{tcl} +% For the replacement with lines to insert, it is necessary to +% figure out the source and extracted line prefixes. These are taken +% from the |from| line of the source, which is the line \emph{after} +% the new lines if this is a pure insertion. +% \begin{tcl} + if {[info exists from]} then { + set sprefix [lindex $lift($n0) 1] + set eprefix [lindex $lift($n0) 2] + } elseif {[info exists lift([lindex $repl 0])]} then { + foreach {from sprefix eprefix} $lift([lindex $repl 0])\ + break + set to [expr {$from-1}] + } else { + incr misapplied [llength [lrange $repl 2 end]] + continue + } + set eplen [string length $eprefix] + set epend [expr {$eplen-1}] +% \end{tcl} +% Actually replacing the lines is pretty straightforward, but the +% |lreplace| command doing this is built dynamically. +% \begin{tcl} + set cmd [list lreplace $SL $from $to] + foreach line [lrange $repl 2 end] { + if {$eprefix eq [string range $line 0 $epend]} then { + lappend cmd "$sprefix[string range $line $eplen end]" + } else { + lappend cmd $line + } + incr applied + } + set SL [eval $cmd] + } +% \end{tcl} +% Only hunks with misapplied lines get included in the log. +% \begin{tcl} + if {$misapplied>0} then { + if {$applied>0} then { + lset hunk 5 "(-- was partially applied --)" + } else { + lset hunk 5 "(not applied)" + } + lappend log $hunk + } + } +% \end{tcl} +% Finally the log is formatted for return. +% \begin{tcl} + set res "" + foreach hunk [lsort -index 0 -integer $log] { + foreach {start1 end1 start2 end2 lines msg} $hunk break + append res [format "@@ -%d,%d +%d,%d @@ %s\n"\ + $start1 [expr {$end1-$start1+1}]\ + $start2 [expr {$end2-$start2+1}] $msg] + foreach {type line} $lines { + switch -- $type 0 { + append res " " $line \n + } - - + { + append res $type $line \n + } + } + } + return $res +} +%</utilpkg> +% \end{tcl} +% +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::patch] [arg source-var] [arg terminals]\ + [arg fromtext] [arg diff] [opt "[arg option] [arg value] ..."]] + This command tries to apply a [syscmd diff] file (for example a + contributed patch) that was computed for a generated file to the + [syscmd docstrip] source. This can be useful if someone has + edited a generated file, thus mistaking it for being the source. + This command makes no presumptions which are specific for the case + that the generated file is a Tcl script. + [para] + + [cmd patch] requires that the source file to patch is kept as a + list of lines in a variable, and the name of that variable in the + calling context is what goes into the [arg source-var] argument. + The [arg terminals] is the list of terminals used to extract the + file that has been patched. The [arg diff] is the actual diff to + apply (in a format as explained below) and the [arg fromtext] is + the contents of the file which served as "from" when the diff was + computed. Options can be used to further control the process. + [para] + + The process works by "lifting" the hunks in the [arg diff] from + generated to source file, and then applying them to the elements of + the [arg source-var]. In order to do this lifting, it is necessary + to determine how lines in the [arg fromtext] correspond to elements + of the [arg source-var], and that is where the [arg terminals] come + in; the source is first [cmd extract]ed under the given + [arg terminals], and the result of that is then matched against + the [arg fromtext]. This produces a map which translates line + numbers stated in the [arg diff] to element numbers in + [arg source-var], which is what is needed to lift the hunks. + [para] + + The reason that both the [arg terminals] and the [arg fromtext] + must be given is twofold. First, it is very difficult to keep track + of how many lines of preamble are supplied some other way than by + copying lines from source files. Second, a generated file might + contain material from several source files. Both make it impossible + to predict what line number an extracted file would have in the + generated file, so instead the algorithm for computing the line + number map looks for a block of lines in the [arg fromtext] which + matches what can be extracted from the source. This matching is + affected by the following options: + [list_begin options] + [opt_def -matching [arg mode]] + How equal must two lines be in order to match? The supported + [arg mode]s are: + [list_begin definitions] + [def [const exact]] + Lines must be equal as strings. This is the default. + [def [const anyspace]] + All sequences of whitespace characters are converted to single + spaces before comparing. + [def [const nonspace]] + Only non-whitespace characters are considered when comparing. + [def [const none]] + Any two lines are considered to be equal. + [list_end] + [opt_def -metaprefix [arg string]] + The [option -metaprefix] value to use when extracting. Defaults + to "%%", but for Tcl code it is more likely that "#" or "##" had + been used for the generated file. + [opt_def -trimlines [arg boolean]] + The [option -trimlines] value to use when extracting. Defaults to + true. + [list_end] + + The return value is in the form of a unified diff, containing only + those hunks which were not applied or were only partially applied; + a comment in the header of each hunk specifies which case is at + hand. It is normally necessary to manually review both the return + value from [cmd patch] and the patched text itself, as this command + cannot adjust comment lines to match new content. + [para] + + An example use would look like +[example_begin] +set sourceL [lb]split [lb]docstrip::util::thefile from.dtx[rb] \n[rb] +set terminals {foo bar baz} +set fromtext [lb]docstrip::util::thefile from.tcl[rb] +set difftext [lb]exec diff --unified from.tcl to.tcl[rb] +set leftover [lb]docstrip::util::patch sourceL $terminals $fromtext\ + [lb]docstrip::util::import_unidiff $difftext[rb] -metaprefix {#}[rb] +set F [lb]open to.dtx w[rb]; puts $F [lb]join $sourceL \n[rb]; close $F +return $leftover +[example_end] + Here, [file from.dtx] was used as source for [file from.tcl], which + someone modified into [file to.tcl]. We're trying to construct a + [file to.dtx] which can be used as source for [file to.tcl]. +%</utilman> +% \end{macrocode} +% \end{proc} +% +% +% +% +% \section{Reading files} +% +% \subsection{Raw file contents} +% +% \begin{proc}{thefile} +% When experimenting with docstripping, it is often convenient to have +% an easy command for reading the contents of a file. The |thefile| +% command (named vaugely in the tradition of such \LaTeX\ commands as +% |\thepage|) returns the contents of the file whose name it is given. +% \changes{1.2}{2005/06/19}{Procedure added. (LH)} +% More precisely, the syntax is +% \begin{quote} +% |docstrip::util::thefile| \word{file name} +% \begin{regblock}[\regstar]\word{option} \word{value}\end{regblock} +% \end{quote} +% where the \word{option} \word{value} pairs are handed to |fconfigure| +% to configure the file before reading it. +% \changes{1.2}{2005/09/07}{Added error handling. (LH)} +% \changes{1.3}{2010/04/12}{Added \texttt{-nonewline} switch. (LH)} +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::thefile {fname args} { + set F [open $fname r] + if {[llength $args]} then { + if {[set code [ + catch {eval [linsert $args 0 fconfigure $F]} res + ]]} then { + close $F + return -code $code -errorinfo $::errorInfo -errorcode\ + $::errorCode + } + } + catch {read -nonewline $F} res + close $F + return $res +} +%</utilpkg> +% \end{tcl} +% The code is thus very straightforward---what remains is to make a +% manpage entry for it. +% \begin{tcl} +%<*utilman> +[call [cmd docstrip::util::thefile] [arg filename] [ + opt "[arg option] [arg value] ..." +]] + The [cmd thefile] command opens the file [arg filename], reads it to + end, closes it, and returns the contents (dropping a final newline + if there is one). The option-value pairs are + passed on to [cmd fconfigure] to configure the open file channel + before anything is read from it. +%</utilman> +% \end{tcl} +% \end{proc} +% +% Better provide some tests too\dots +% \begin{tcl} +%<*utiltest> +tcltest::test docstrip::util::thefile-1.1 {thefile without args}\ + -setup { + set Fname [tcltest::makeFile [ + join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + } \n + ] test.txt] +} -body { + docstrip::util::thefile $Fname +} -cleanup { + tcltest::removeFile $Fname +} -result [join { + {% Just a minor test file.} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} +} \n] +% \end{tcl} +% This one tests that an error in the number of arguments is caught +% correctly. +% \begin{tcl} +tcltest::test docstrip::util::thefile-1.2 {thefile with wrong no. args}\ + -setup { + set Fname [tcltest::makeFile [ + join { + {% Just a minor test file (contents irrelevant).} + {puts A} + {%<*bar>} + {puts B} + {%<*foo>} + {puts [info exists baz]} + } \n + ] test.txt] +} -body { + docstrip::util::thefile $Fname -translation binary -buffering +} -cleanup { + tcltest::removeFile $Fname +} -returnCodes error +% \end{tcl} +% This one tests configuring (of encoding). +% \begin{tcl} +tcltest::test docstrip::util::thefile-1.3 {thefile with args} -setup { + set Fname [tcltest::makeFile "Dummy content to overwrite" test.xxx] + set F [open $Fname w] + fconfigure $F -translation binary + puts -nonewline $F [encoding convertto utf-8 \u00E5\u00E4\u00F6] + close $F +} -body { + docstrip::util::thefile $Fname -encoding utf-8 +} -cleanup { + tcltest::removeFile $Fname +} -result \u00E5\u00E4\u00F6 +%</utiltest> +% \end{tcl} +% +% +% \subsection{The diff format} +% \label{Ssec:Tcldiff} +% +% The difference format used by |docstrip::util::patch| is a +% \Tcllogo-list format into which the common diff formats can be parsed. +% Each hunk is a five element list +% \begin{quote} +% \word{start1} \word{end1} \word{start2} \word{end2} \word{lines} +% \end{quote} +% where the start and end elements are integers, specifying the line +% numbers of the first and last lines in the hunk respectively. 1 +% elements pertain to the first file and 2 elements to the second file. +% The number of the first line in a file is |1|. +% +% The \word{lines} is a list of the form +% \begin{quote} +% \begin{regblock}[\regplus]\word{type} \word{text}\end{regblock} +% \end{quote} +% where each \word{text} is an actual line of text (minus newline) and +% the \word{type} specifies the type of this line. |+| means a line that +% is in the second file but not the first, |-| means a line that is in +% the first file but not the second, and |0| means a line that is in +% both files. The format is thus most similar to the \texttt{--unified} +% diff format, but the difference is not too great to the other formats +% either. The \word{type} may also be |synch|, which is used as a +% placeholder for any number of ``invisible'' lines (neither in the +% first or second file, but perhaps present in the source) at that +% point. The \word{text} of |synch| lines is ignored. +% +% As an example of what it looks like, a difference between the two +% files +%\begin{verbatim} +%foo +%bar baz +%end +%\end{verbatim} +% and +%\begin{verbatim} +%foo +%bar +%baz +%end +%\end{verbatim} +% is +% \begin{quote} +% |1 3 1 4 {0 foo - {bar baz} + bar + baz 0 end}| +% \end{quote} +% An alternative is +% \begin{quote} +% |2 2 2 3 {+ bar - {bar baz} + baz}| +% \end{quote} +% since \texttt{+} lines ``commute'' with \texttt{-} lines. +% +% \begin{proc}{import_unidiff} +% The |import_unidiff| procedure imports a standard diff in unified +% format to the format described above. The call syntax is +% \begin{quote} +% |docstrip::util::import_unidiff| \word{diff-text} +% \word{warning-var}\regopt +% \end{quote} +% where the \word{diff-text} is the actual text of the diff file to +% convert, and the \word{warning-var} is the name of a variable in +% the calling context to which warnings about failings to parse the +% input \word{diff-text} will be appended. The return format is a +% list as described above. +% +% In the implementation, the hunk-to-be is kept in the five +% variables |start1|, |end1|, |start2|, |end2|, and |lines|. +% Whether |end2| is an integer is used as a signal for whether there +% is a hunk to append. Malformed hunk headers will cause that hunk +% to be ignored. +% \begin{tcl} +%<*utilpkg> +proc docstrip::util::import_unidiff {text {warnvar ""}} { + if {$warnvar ne ""} then {upvar 1 $warnvar warning} + set inheader 1 + set res [list] + set lines [list] + set end2 "not an integer" + foreach line [split $text \n] { + if {$inheader && [regexp {^(---|\+\+\+)} $line]}\ + then {continue} + switch -glob -- $line { *} { + lappend lines 0 [string range $line 1 end] + } {+*} { + lappend lines + [string range $line 1 end] + } {-*} { + lappend lines - [string range $line 1 end] + } @@* { + if {[string is integer $end2]} then { + lappend res [list $start1 $end1 $start2 $end2 $lines] + } + set len2 [set len1 ,1] + if {[ + regexp {^@@ -([0-9]+)(,[0-9]+)? \+([0-9]+)(,[0-9]+)? @@}\ + $line -> start1 len1 start2 len2 + ] && [scan "$start1 $len1,1" {%d ,%d} start1 len1]==2 &&\ + [scan "$start2 $len2,1" {%d ,%d} start2 len2]==2 + } then { + set end1 [expr {$start1+$len1-1}] + set end2 [expr {$start2+$len2-1}] + set inheader 0 + } else { + set end2 "not an integer" + append warning "Could not parse hunk header: " $line \n + } + set lines [list] + } "" { +% \end{tcl} +% Empty lines are ignored (there will typically be one at the end of +% the |foreach| loop). +% \begin{tcl} + } default { + append warning "Could not parse line: " $line \n + } + } + if {[string is integer $end2]} then { + lappend res [list $start1 $end1 $start2 $end2 $lines] + } + return $res +} +%</utilpkg> +% \end{tcl} +% +% \begin{macrocode} +%<*utilman> +[call [cmd docstrip::util::import_unidiff] [arg diff-text]\ + [opt [arg warning-var]]] + This command parses a unified ([syscmd diff] flags [option -U] and + [option --unified]) format diff into the list-of-hunks format + expected by [cmd docstrip::util::patch]. The [arg diff-text] + argument is the text to parse and the [arg warning-var] is, if + specified, the name in the calling context of a variable to which + any warnings about parsing problems will be [cmd append]ed. + [para] + + The return value is a list of [term hunks]. Each hunk is a list of + five elements "[arg start1] [arg end1] [arg start2] [arg end2] + [arg lines]". [arg start1] and [arg end1] are line numbers in the + "from" file of the first and last respectively lines of the hunk. + [arg start2] and [arg end2] are the corresponding line numbers in + the "to" file. Line numbers start at 1. The [arg lines] is a list + with two elements for each line in the hunk; the first specifies the + type of a line and the second is the actual line contents. The type + is [const -] for lines only in the "from" file, [const +] for lines + that are only in the "to" file, and [const 0] for lines that are + in both. +[list_end] +%</utilman> +% \end{macrocode} +% \end{proc} +% +% +% +% \section{Closing material} +% +% The packages need no particular ending, but the tests can do with an +% explicit cleanup. +% +% \begin{tcl} +%<*test,utiltest> +%<!tcllibtest>tcltest::cleanupTests +%<tcllibtest>testsuiteCleanup +%</test,utiltest> +% \end{tcl} +% +% The manpages require an explicit ending, and can do with some +% keywords. +% \begin{macrocode} +%<*man,utilman> +[manpage_end] +%</man,utilman> +% \end{macrocode} +% There! That's it! +% +% +% \section{Development tools} +% +% I have found the following code snippets useful for formatting +% \texttt{docstrip.man}. +% \begin{tcl} +%<*devmantest> +package require doctools +doctools::new man2html -format html +proc makehtml {{from docstrip.man} {to docstrip.html}} { + set text [string map {\r \n}\ + [getText -w $from [minPos] [maxPos -w $from]]] + set html [man2html format $text] + replaceText -w $to [minPos] [maxPos -w $to]\ + [string map {\n \r} $html] +} +proc dtx2html {terminals {to docstrip_util.html} {from tcldocstrip.dtx}} { + set text [string map {\r \n}\ + [getText -w $from [minPos] [maxPos -w $from]]] + set html [man2html format [docstrip::extract $text $terminals]] + replaceText -w $to [minPos] [maxPos -w $to]\ + [string map {\n \r} $html] +} +%</devmantest> +% \end{tcl} +% It is included here so that I know where to find it, but it is +% normally no extracted. +% +% \bigskip +% +% The following block of code could be taken as the beginnings of a test +% or example of the use of |ddt2man|. First extract the +% \Module{gcdexample}. +% \begin{tcl} +%<*devtest2> +package require docstrip +set F [open tcldocstrip.dtx r] +set text [docstrip::extract [read $F] gcdexample] +close $F +% \end{tcl} +% Then unindent the lines so that they become the intended mixture of +% code and comment lines. +% \begin{tcl} +regsub -all -lineanchor {^ } $text "" ddt +% \end{tcl} +% Now |ddt2html| can be applied: +% \begin{tcl} +package require docstrip::util +set man [docstrip::util::ddt2man $ddt] +% \end{tcl} +% Finally, format this code as something. +% \begin{tcl} +package require doctools +doctools::new man2html -format html +set html [man2html format $man] +%</devtest2> +% \end{tcl} +% +% \begin{thebibliography}{6} +% \bibitem{tclldoc} +% Lars Hellstr\"om: +% \textit{The \textsf{tclldoc} package and class}, +% \LaTeXe\ package and document class, +% \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash +% \texttt{latex}\slash \texttt{contrib}\slash \texttt{tclldoc}/. +% \bibitem{doctools_fmt} +% Andreas Kupries: +% \textit{Specification of a simple \Tcllogo\ Markup Language +% for Manpages}, manpage, +% \texttt{tcllib} module \textsf{doctools}, 2002--; +% \textsc{http}:/\slash \texttt{core.tcl.tk/tcllib}\slash +% \texttt{doc}\slash \texttt{doctools\_fmt.html}. +% \bibitem{docstrip} +% Frank Mittelbach, Denys Duchier, Johannes Braams, Marcin +% Woli\'nski, and Mark Wooding: \textit{The \textsf{DocStrip} +% program}, The \LaTeX3 Project; +% \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash +% \texttt{latex}\slash \texttt{base}\slash \texttt{docstrip.dtx}. +% \bibitem{doc} +% Frank Mittelbach, B.~Hamilton Kelly, Andrew Mills, Dave Love, and +% Joachim \mbox{Schrod}: \textit{The \textsf{doc} and +% \textsf{shortvrb} Packages}, The \LaTeX3 Project; +% \textsc{ctan}:\discretionary{}{}{\thinspace}\texttt{macros}\slash +% \texttt{latex}\slash \texttt{base}\slash \texttt{doc.dtx}. +% \iffalse +% [enum] +% Chapter 14 of +% [emph {The LaTeX Companion}] (second edition), +% Addison-Wesley, 2004; ISBN 0-201-36299-6. +% \fi +% \end{thebibliography} +% +% +\endinput + +
\ No newline at end of file diff --git a/tcllib/modules/docstrip/tcldocstrip.ins b/tcllib/modules/docstrip/tcldocstrip.ins new file mode 100644 index 0000000..2eae3fd --- /dev/null +++ b/tcllib/modules/docstrip/tcldocstrip.ins @@ -0,0 +1,46 @@ +% tcldocstrip.ins --- DOCSTRIP installation script for +% the docstrip Tcl package +\input docstrip + +% Redefine the \MetaPrefix; it should be something which starts a +% until-end-of-line comment: +\edef\MetaPrefix{\string#\string#} + + +% Redefine the file preamble and postamble; this is necessary because +% otherwise the old \metaPrefix is inserted at the beginning of these +% lines. +\preamble + +In other words: +************************************** +* This Source is not the True Source * +************************************** +the true source is the file from which this one was generated. + +\endpreamble + +\postamble +\endpostamble + +\askforoverwritefalse + +% Actually make docstrip.tcl et al.: +\generate{ + \file{docstrip.tcl} {\from{tcldocstrip.dtx}{pkg}} + \file{docstrip_util.tcl} {\from{tcldocstrip.dtx}{utilpkg}} + \file{pkgIndex.tcl} {\from{tcldocstrip.dtx}{idx}} + \file{docstrip.test} {\from{tcldocstrip.dtx}{test}} + \file{docstrip_util.test}{\from{tcldocstrip.dtx}{utiltest}} + % The .test files are generated with an extra option tcllibtest + % by tcldocstrip.stitch. This causes them to make use of code + % that is unlikely to be present outside the tcllib test + % environment. + \usepreamble\empty + \usepostamble\empty + \file{docstrip.man} {\from{tcldocstrip.dtx}{man}} + \file{docstrip_util.man} {\from{tcldocstrip.dtx}{utilman}} +} + + +\end diff --git a/tcllib/modules/docstrip/tcldocstrip.stitch b/tcllib/modules/docstrip/tcldocstrip.stitch new file mode 100644 index 0000000..800eeae --- /dev/null +++ b/tcllib/modules/docstrip/tcldocstrip.stitch @@ -0,0 +1,25 @@ +# -*- tcl -*- +# Stitch definition for docstrip files, used by SAK. + +input tcldocstrip.dtx + +options -metaprefix \# -preamble {In other words: +************************************** +* This Source is not the True Source * +************************************** +the true source is the file from which this one was generated. +} + +stitch docstrip.tcl pkg +stitch docstrip_util.tcl utilpkg +stitch pkgIndex.tcl idx +stitch docstrip.test {test tcllibtest} +stitch docstrip_util.test {utiltest tcllibtest} +# For the .test files, the tcllibtest guard is not present in +# the corresponding commmand in tcldocstrip.ins. The rationale +# for this is that someone using the .ins rather than the .stitch +# is unlikely to have a tcllib testing environment at hand. + +options -nopreamble -nopostamble +stitch docstrip.man man +stitch docstrip_util.man utilman |