From b9d10d08c4eb0dedaea3b1bcde0f13b033e16c85 Mon Sep 17 00:00:00 2001
From: Alexander Belopolsky <alexander.belopolsky@gmail.com>
Date: Thu, 11 Nov 2010 14:07:41 +0000
Subject: Issue #10386: Added __all__ to token module; this simplifies
 importing in tokenize module and prevents leaking of private names through
 import *.

---
 Lib/token.py    | 18 ++++++++----------
 Lib/tokenize.py |  5 ++---
 Misc/NEWS       |  4 ++++
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Lib/token.py b/Lib/token.py
index a48bf3c..7e2bfcf 100755
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -1,7 +1,7 @@
-#! /usr/bin/env python3
-
 """Token constants (from "token.h")."""
 
+__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
+
 #  This file is automatically generated; please don't muck it up!
 #
 #  To update the symbols in this file, 'cd' to the top directory of
@@ -68,12 +68,10 @@ N_TOKENS = 55
 NT_OFFSET = 256
 #--end constants--
 
-tok_name = {}
-for _name, _value in list(globals().items()):
-    if type(_value) is type(0):
-        tok_name[_value] = _name
-del _name, _value
-
+tok_name = {value: name
+            for name, value in globals().items()
+            if isinstance(value, int)}
+__all__.extend(tok_name.values())
 
 def ISTERMINAL(x):
     return x < NT_OFFSET
@@ -85,7 +83,7 @@ def ISEOF(x):
     return x == ENDMARKER
 
 
-def main():
+def _main():
     import re
     import sys
     args = sys.argv[1:]
@@ -139,4 +137,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    _main()
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 7745412..506aa6a 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -33,9 +33,8 @@ from io import TextIOWrapper
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
-__all__ = [x for x in dir(token) if not x.startswith("_")]
-__all__.extend(["COMMENT", "tokenize", "detect_encoding", "NL", "untokenize",
-                "ENCODING", "TokenInfo"])
+__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
+                           "NL", "untokenize", "ENCODING", "TokenInfo"]
 del token
 
 COMMENT = N_TOKENS
diff --git a/Misc/NEWS b/Misc/NEWS
index 7dc9b6c..a887faf 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -63,6 +63,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #10386: Add __all__ to token module; this simplifies importing
+  in tokenize module and prevents leaking of private names through
+  import *.
+
 - Issue #4471: Properly shutdown socket in IMAP.shutdown().  Patch by
   Lorenzo M. Catucci.
 
-- 
cgit v0.12