Merge commit 'e7b18aa8819d8e21e23fa80456c80f50feaef1c7' as 'fickle'

9 files changed, 522 insertions, 0 deletions

diff --git a/fickle/examples/Makefile b/fickle/examples/Makefile
new file mode 100644
index 0000000..4a40f68
--- /dev/null
+++ b/fickle/examples/Makefile
@@ -0,0 +1,20 @@
+# $Id: Makefile,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+# A simple Makefile that calls fickle upon the example specification
+# files.
+
+TCL=/usr/bin/tclsh
+FICKLE=../fickle.tcl
+FCL_EXS=cat.tcl verbs.tcl wc.tcl wc2.tcl cl.tcl csa.tcl tsa.tcl
+
+all: fcl_exs
+
+fcl_exs: $(FCL_EXS)
+
+%.tcl: %.fcl
+	-$(TCL) $(FICKLE) $<
+
+clean:
+	-rm -f $(FCL_EXS:.fcl=.tcl)
+
+.PHONY: clean
diff --git a/fickle/examples/README b/fickle/examples/README
new file mode 100644
index 0000000..117e566
--- /dev/null
+++ b/fickle/examples/README
@@ -0,0 +1,102 @@
+$Id: README,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+The example fickle code programs are based upon the lex examples found
+within "lex & yacc" by John R. Levine, Tony Mason, and Doug Brown (by
+O'Reilly & Associates, ISBN 1-56592-000-7).  For more information on
+using lex and yacc, see http://www.oreilly.com/catalog/lex/.
+
+Run the Makefile to generate resulting Tcl code.  Descriptions of
+individual files are below.  The reader is assumed to have a familiarity
+with flex; if not consider purchasing the aforementioned book.
+
+
+cat.fcl
+-------
+This is the simplest fickle example possible.  It copies its input (from
+stdin) to output (stdout), much like the cat(1) program does without any
+arguments.  Note that one must explicitly call yylex to run the lexer.
+
+
+verbs.fcl
+---------
+This examples demonstrates a verbatim section (the text between '%{'
+and '%}').  Also note how fickle specification files may have
+Tcl-style comments embedded within.  This program searches its input
+for various English verbs and copies it to the output.  The program
+makes use of the variable $yytext (really just an upvar'ed version of
+the global ::yytext variable) to print out the text that matched.
+
+
+wc.fcl
+------
+This program will count the number of characters, words, and lines in
+its input, much like wc(1).  Called without any arguments, wc.tcl reads
+from stdin; it reads from a file if given an argument.  Unless otherwise
+specified, 'yyin' points to stdin and 'yyout' to stdout.  Overriding
+these variables forces the lexer to read from another stream.  This
+program also uses $yyleng (an upvar'ed version of ::yyleng) which is
+set to [string length $::yytext].
+
+
+wc2.fcl
+-------
+This example supports multiple filenames.  With more than one argument,
+wc2 will print a report for each line and a summary line totalling all
+metrics.  No summary is displayed given zero or one argument.
+
+wc2 handles multiple files by overriding the definition for yywrap.
+yywrap returns 0 when additional files need to be processed.
+
+The directive `%option noheaders' causes fickle to not include
+comments about the autogenerated functions within wc2.tcl.  Note the
+difference in size between wc2.tcl and wc.tcl.
+
+
+cl.fcl
+------
+This example demonstrates how to feed the lexer input from a source
+other than a file -- in this case, the command line.  One must rewrite
+YY_INPUT to use the alternative source.  The first parameter should be
+'upvar'ed; it holds the next string to scan.  'result' should be the
+size of the buffer, or zero to indicate an end of file; this too needs
+to be 'upvar'ed.  The final parameter indicates the maximum allowed
+buffer size.  By default this is 1024; use the `%option buffersize'
+directive to change this amount.
+
+Also note the use of `%option nodefault'.  By default fickle will
+write to yyout any unmatched input through the ECHO function.  Use
+`%option nodefault' to abort the program upon unmatched input; this is
+useful during debugging sessions.  One can also invoke this
+suppression behavior with the `-s' flag on the fickle command line.
+
+
+csa.fcl
+-------
+The next example is a C source analyzer.  It takes a single C source
+file as a parameter; it then counts the lines of code, comments, and
+whitespace within.
+
+This example demonstrates the start state feature of fickle, enabled
+through the directive `%option stack'.  fickle supports both exclusive
+start states (as indicated by '%x') as well as regular start states
+('%s', though not featured in this program).  Start states specify
+when a pattern is allowed.  Switch states through calls to 'BEGIN',
+'yy_push_state', and 'yy_pop_state'.
+
+The initial state is called, not surprisingly, 'INITIAL'.  Unlike flex,
+'BEGIN 0' and 'BEGIN INITIAL' are not identical.  To match all states,
+prepend the pattern with '<*>'.  Patterns that have no state listed are
+defaulted to matching only INITIAL and any regular start state.[*]
+
+Note that if multiple patterns match the input, the largest match takes
+precedence.  In case of a tie the pattern appearing earlier within the
+specification file wins.
+
+[*] Regular start states are a source of much confusion and are rarely
+useful.  Avoid them like the plague.
+
+
+tsa.fcl
+-------
+In comparison to the above this program instead analyzes Tcl code.
+It's not particularly foolproof but does get the job done.
diff --git a/fickle/examples/cat.fcl b/fickle/examples/cat.fcl
new file mode 100644
index 0000000..ebbd8e9
--- /dev/null
+++ b/fickle/examples/cat.fcl
@@ -0,0 +1,4 @@
+%%
+.|\n	ECHO;
+%%
+yylex
diff --git a/fickle/examples/cl.fcl b/fickle/examples/cl.fcl
new file mode 100644
index 0000000..0949843
--- /dev/null
+++ b/fickle/examples/cl.fcl
@@ -0,0 +1,76 @@
+# $Id: cl.fcl,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+# Scans its command line for various arguments.
+
+# This is based upon example 'ape-05.l' (which is the flex version of
+# 'ch2-05.l') from "lex & yacc" by John R. Levine, Tony Mason, and
+# Doug Brown (by O'Reilly & Associates, ISBN 1-56592-000-7).  For more
+# information on using lex and yacc, see
+# http://www.oreilly.com/catalog/lex/.
+
+# myinput() could have been written much more efficiently because Tcl
+# handles command line arguments as a list.  For the sake of porting
+# the original example to Tcl, I used the same logic found within the
+# original flex code.
+
+%{
+#!/usr/bin/tclsh
+%}
+
+%buffersize 1024
+%option nodefault
+
+%%
+
+-h          |
+-\?         |
+-help       {
+              puts "usage is: $::progName \[-help | -h | -? \] \[-verbose | -v \] \[(-file | -f) filename\]"
+              # actually, the -f option is not handled by this program.
+              # that is left as an exercise to the reader.
+              }
+-v          |
+-verbose    {
+              puts "verbose mode is on"
+              set ::verbose 1
+            }
+
+%%
+
+proc YY_INPUT {buf result max} {
+    upvar $result ret_val
+    upvar $buf buf_data
+    set ret_val [myinput buf_data $max]
+}
+
+set ::offset 0
+proc myinput {buf max} {
+    upvar $buf buf_data
+    if {[llength $::targv] == 0} {
+        # no arguments left, so return an EOF
+        return 0
+    }
+    set len [string length [lindex $::targv 0]]
+    if {$len >= $max} {
+        set copylen [expr {$max - 1}]
+    } else {
+        set copylen $len
+    }
+    if {$len > 0} {
+        set buf_data [string range [lindex $::targv 0] $::offset $copylen]
+    }
+    if {[string length [lindex $::targv 0]] >= $::offset + $copylen} {
+        append buf " "
+        incr copylen
+        set ::offset 0
+        set ::targv [lrange $::targv 1 end]
+    } else {
+        incr ::offset $copylen
+    }
+    return $copylen
+}
+
+set progName $argv0
+set verbose 0
+set ::targv $argv                 ;# holds remainder of argument list
+yylex
diff --git a/fickle/examples/csa.fcl b/fickle/examples/csa.fcl
new file mode 100644
index 0000000..3cd6dc3
--- /dev/null
+++ b/fickle/examples/csa.fcl
@@ -0,0 +1,65 @@
+# $Id: csa.fcl,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+# Counts the lines of comments, code, and whitespace within a C
+# program.
+
+# This is based upon example 'ch2-09.l' from "lex & yacc" by John
+# R. Levine, Tony Mason, and Doug Brown (by O'Reilly & Associates, ISBN
+# 1-56592-000-7).  For more information on using lex and yacc, see
+# http://www.oreilly.com/catalog/lex/.
+
+%{
+#!/usr/bin/tclsh
+
+set comments 0
+set code 0
+set whitespace 0
+
+proc update_count { a b c } {
+    incr ::comments $a
+    incr ::code $b
+    incr ::whitespace $c
+    puts -nonewline "code: $::code, comments: $::comments, whitespace: $::whitespace\r"
+    flush stdout
+}
+
+%}
+
+%option noheaders stack nodefault
+%x COMMENT
+
+%%
+
+^[ \t]*"/*"                 { BEGIN COMMENT }
+^[ \t]*"/*".*"*/"[ \t]*\n   { update_count 1 0 0 }
+<COMMENT>"*/"[ \t]*\n       { BEGIN INITIAL; update_count 1 0 0 }
+<COMMENT>"*/"               { BEGIN INITIAL }
+<COMMENT>\n                 { update_count 1 0 0 }
+<COMMENT>.\n                { update_count 1 0 0 }
+
+^[ \t]*\n                   { update_count 0 0 1 }
+
+.+"/*".*"*/".*\n            { update_count 0 1 0 }
+.*"/*".*"*/".+\n            { update_count 0 1 0 }
+.+"/*".*\n                  { BEGIN COMMENT; update_count 0 1 0 }
+.\n                         { update_count 0 1 0 }
+
+<*>.                        # do nothing
+
+%%
+
+if {[llength $argv] == 0} {
+    puts stderr "C source analyzer needs a filename."
+    exit 0
+}
+
+if {[catch {open [lindex $argv 0] r} yyin]} {
+    puts stderr "Could not open [lindex $argv 0]"
+    exit 0
+}
+
+yylex
+
+close $yyin
+
+puts ""
diff --git a/fickle/examples/tsa.fcl b/fickle/examples/tsa.fcl
new file mode 100644
index 0000000..86a3733
--- /dev/null
+++ b/fickle/examples/tsa.fcl
@@ -0,0 +1,79 @@
+#$Id: tsa.fcl,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+# Counts lines of comments, logical lines of code, and function
+# invocations in Tcl code.
+
+# The patterns can handle most "normal" Tcl code.  There are some
+# instances where it will not correctly detect a function call.
+
+%{
+#!/usr/bin/tclsh
+
+proc found_func {funcname} {
+    if [info exist ::func($funcname)] {
+        incr ::func($funcname)
+    } else {
+        set ::func($funcname) 1
+    }
+}
+
+proc spin {} {
+    if {$::numlines % 8 == 0} {
+        puts -nonewline "."
+        flush stdout
+    }
+}
+
+set comments 0
+set numlines 0
+set spinner_count 0
+
+%}
+
+%option stack
+%x ARG
+
+%%
+
+<*>^\s*\n            { incr ::numlines; spin }
+<*>;?\s*#.*\n        { incr ::comments; incr ::numlines; spin }
+<*>\n                { yy_pop_state; incr ::numlines; spin }
+<*>\s                # ignore whitespace
+<*>\\(.|\n)          # ignore escaped characters
+<*>\d+               # numbers are ignored
+<INITIAL>\w+         { found_func $yytext; yy_push_state ARG }
+<ARG>\w+             # ignore arguments
+<*>\[\s*\w+          { set start [string first "\["  $yytext]
+                       set func [string range $yytext [expr {$start + 1}] end]
+                       found_func [string trim $func]
+                       yy_push_state ARG }
+<ARG>\]              { yy_pop_state }
+<*>;                 { yy_pop_state }
+<*>.                 # unknown character; ignore it
+
+%%
+
+# start of main
+if {[llength $argv] > 0} {
+    if {[catch {open [lindex $argv 0]} yyin]} {
+        puts stderr "could not open file"
+        exit 0
+    }
+}
+
+yylex
+
+if {[llength $argv] > 0} {
+    close $yyin
+}
+
+puts ""
+puts "Comments:  $comments"
+puts "Num lines: $numlines"
+puts "Function calls:"
+parray func
+set totalcalls 0
+foreach {name calls} [array get func] {
+    incr totalcalls $calls
+}
+puts "Total calls: $totalcalls"
diff --git a/fickle/examples/verbs.fcl b/fickle/examples/verbs.fcl
new file mode 100644
index 0000000..c9317ae
--- /dev/null
+++ b/fickle/examples/verbs.fcl
@@ -0,0 +1,34 @@
+# $Id: verbs.fcl,v 1.1.1.1 2004/07/23 19:22:41 tang Exp $
+
+# Recognizes various English verbs in sentences.
+
+# This is based upon example 'ch1-02.l' from "lex & yacc" by John
+# R. Levine, Tony Mason, and Doug Brown (by O'Reilly & Associates, ISBN
+# 1-56592-000-7).  For more information on using lex and yacc, see
+# http://www.oreilly.com/catalog/lex/.
+
+%{
+#!/usr/bin/tclsh
+%}
+
+%%
+[\t ]+        # ignore whitespace
+is |
+am |
+are |
+were |
+was |
+be |
+being |
+been |
+do |
+does |
+did |
+will          puts "$yytext: is a verb"
+[a-zA-Z]+     puts "$yytext: is not a verb"
+
+.|\n          ECHO  ;# normal default anyway
+
+%%
+
+yylex
diff --git a/fickle/examples/wc.fcl b/fickle/examples/wc.fcl
new file mode 100644
index 0000000..871b3ad
--- /dev/null
+++ b/fickle/examples/wc.fcl
@@ -0,0 +1,40 @@
+%{
+#!/usr/bin/tclsh
+
+# Counts characters, words, and lines within its input.
+
+# This is based upon example 'ch2-02.l' from "lex & yacc" by John
+# R. Levine, Tony Mason, and Doug Brown (by O'Reilly & Associates, ISBN
+# 1-56592-000-7).  For more information on using lex and yacc, see
+# http://www.oreilly.com/catalog/lex/.
+
+set charCount 0
+set wordCount 0
+set lineCount 0
+
+%}
+
+word  [^ \t\n]+
+eol   \n
+
+%%
+
+{word}   { incr ::wordCount; incr ::charCount $yyleng }
+{eol}    { incr ::charCount; incr ::lineCount }
+.        { incr ::charCount }
+
+%%
+
+if {[llength $argv] > 0} {
+    if {[catch {open [lindex $argv 0]} f]} {
+        puts stderr "could not open file [lindex $argv 0]"
+        exit 1
+    }
+    set yyin $f
+}
+
+yylex
+
+puts "$charCount $wordCount $lineCount"
+
+return 0
diff --git a/fickle/examples/wc2.fcl b/fickle/examples/wc2.fcl
new file mode 100644
index 0000000..e887003
--- /dev/null
+++ b/fickle/examples/wc2.fcl
@@ -0,0 +1,102 @@
+%{
+#!/usr/bin/tclsh
+
+# Counts characters, words, and lines, with support for multiple
+# filenames.
+
+# This is based upon example 'ch2-03.l' from "lex & yacc" by John
+# R. Levine, Tony Mason, and Doug Brown (by O'Reilly & Associates, ISBN
+# 1-56592-000-7).  For more information on using lex and yacc, see
+# http://www.oreilly.com/catalog/lex/.
+
+set charCount 0
+set wordCount 0
+set lineCount 0
+
+%}
+
+%option noheaders
+
+word  [^ \t\n]+
+eol   \n
+
+%%
+
+{word}   { incr ::wordCount; incr ::charCount $yyleng }
+{eol}    { incr ::charCount; incr ::lineCount }
+.        { incr ::charCount }
+
+%%
+
+# lexer calls yywrap to handle EOF conditions (e.g., to
+# connect to a new file, as we do in this case.)
+proc yywrap {} {
+    set file ""
+    if {$::currentFile != 0 && $::nFiles > 1 && $::currentFile < $::nFiles} {
+        # print out statstics for previous file
+        puts [format "%8u %8u %8u %s" $::lineCount $::wordCount $::charCount \
+                [lindex $::fileList [expr {$::currentFile - 1}]]]
+        incr ::totalCC $::charCount
+        incr ::totalWC $::wordCount
+        incr ::totalLC $::lineCount
+        set ::charCount 0
+        set ::wordCount 0
+        set ::lineCount 0
+        close $::yyin
+    }
+    while {$::currentFile < $::nFiles} {
+        if {[catch {open [lindex $::fileList $::currentFile] r} file]} {
+            puts stderr "could not open [lindex $::fileList $::currentFile]"
+            incr ::currentFile
+        } else {
+            set ::yyin $file
+            incr ::currentFile
+            break
+        }
+    }
+    if {$file != ""} {
+        return 0                ;# 0 means there's more input
+    } else {
+        return 1
+    }
+}
+
+set fileList ""
+set currentFile 0
+set nFiles 0
+set totalCC 0
+set totalWC 0
+set totalLC 0
+
+set fileList $argv
+set nFiles [llength $argv]
+
+if {[llength $argv] == 1} {
+    # handle single file case differenly since we don't need to print a
+    # summary line
+    set currentFile 1
+    if {[catch {open [lindex $argv 0] r} file]} {
+        puts stderr "could not open file [lindex $argv 0]"
+        exit 1
+    }
+    set yyin $file
+}
+if {[llength $argv] > 1} {
+    yywrap
+}
+
+yylex
+
+# handle zero or one file differently from multiple files
+if {[llength $argv] > 1} {
+    puts [format "%8u %8u %8u %s" $lineCount $wordCount $charCount \
+            [lindex $argv [expr {$currentFile - 1}]]]
+    incr totalCC $charCount
+    incr totalWC $wordCount
+    incr totalLC $lineCount
+    puts [format "%8u %8u %8u total" $totalLC $totalWC $totalCC]
+} else {
+    puts [format "%8u %8u %8u" $lineCount $wordCount $charCount]
+}
+
+return 0