Merged revisions 62734,62736,62748,62769 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r62734 | brett.cannon | 2008-05-05 22:21:38 +0200 (Mon, 05 May 2008) | 5 lines

  Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
  Ippolito.

  Closes issue #2750.
........
  r62736 | georg.brandl | 2008-05-05 22:53:39 +0200 (Mon, 05 May 2008) | 2 lines

  Fix JSON module docs.
........
  r62748 | benjamin.peterson | 2008-05-06 04:51:10 +0200 (Tue, 06 May 2008) | 2 lines

  PEP 8 nits in json package
........
  r62769 | christian.heimes | 2008-05-06 18:18:41 +0200 (Tue, 06 May 2008) | 2 lines

  Intern static string
  Use float constructors instead of magic code for float constants
........

1 files changed, 69 insertions, 0 deletions

diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py
new file mode 100644
index 0000000..4b065ab
--- /dev/null
+++ b/Lib/json/scanner.py
@@ -0,0 +1,69 @@
+"""Iterator based sre token scanner
+
+"""
+
+import re
+import sre_parse
+import sre_compile
+import sre_constants
+
+from re import VERBOSE, MULTILINE, DOTALL
+from sre_constants import BRANCH, SUBPATTERN
+
+__all__ = ['Scanner', 'pattern']
+
+FLAGS = (VERBOSE | MULTILINE | DOTALL)
+
+class Scanner(object):
+    def __init__(self, lexicon, flags=FLAGS):
+        self.actions = [None]
+        # Combine phrases into a compound pattern
+        s = sre_parse.Pattern()
+        s.flags = flags
+        p = []
+        for idx, token in enumerate(lexicon):
+            phrase = token.pattern
+            try:
+                subpattern = sre_parse.SubPattern(s,
+                    [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
+            except sre_constants.error:
+                raise
+            p.append(subpattern)
+            self.actions.append(token)
+
+        s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
+        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+        self.scanner = sre_compile.compile(p)
+
+    def iterscan(self, string, idx=0, context=None):
+        """Yield match, end_idx for each match
+
+        """
+        match = self.scanner.scanner(string, idx).match
+        actions = self.actions
+        lastend = idx
+        end = len(string)
+        while True:
+            m = match()
+            if m is None:
+                break
+            matchbegin, matchend = m.span()
+            if lastend == matchend:
+                break
+            action = actions[m.lastindex]
+            if action is not None:
+                rval, next_pos = action(m, context)
+                if next_pos is not None and next_pos != matchend:
+                    # "fast forward" the scanner
+                    matchend = next_pos
+                    match = self.scanner.scanner(string, matchend).match
+                yield rval, matchend
+            lastend = matchend
+
+
+def pattern(pattern, flags=FLAGS):
+    def decorator(fn):
+        fn.pattern = pattern
+        fn.regex = re.compile(pattern, flags)
+        return fn
+    return decorator