From 2a58b0636d1f620f8a85a2e4c030cc10551936a5 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 30 May 2019 15:06:32 -0700 Subject: bpo-5028: Fix up rest of documentation for tokenize documenting line (GH-13686) https://bugs.python.org/issue5028 --- Doc/library/tokenize.rst | 4 ++-- Lib/lib2to3/pgen2/tokenize.py | 2 +- Lib/tokenize.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index c89d3d4..b208ba4 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -39,8 +39,8 @@ The primary entry point is a :term:`generator`: column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed (the last tuple item) - is the *physical* line; continuation lines are included. The 5 tuple is - returned as a :term:`named tuple` with the field names: + is the *physical* line. The 5 tuple is returned as a :term:`named tuple` + with the field names: ``type string start end line``. The returned :term:`named tuple` has an additional property named diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 0f9fde3..7924ff3 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -346,7 +346,7 @@ def generate_tokens(readline): column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the - physical line; continuation lines are included. + physical line. """ lnum = parenlev = continued = 0 contstr, needcont = '', 0 diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 738fb71..1aee21b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -415,7 +415,7 @@ def tokenize(readline): column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the - physical line; continuation lines are included. + physical line. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. -- cgit v0.12