Merged revisions 60151-60159,60161-60168,60170,60172-60173,60175 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r60151 | christian.heimes | 2008-01-21 14:11:15 +0100 (Mon, 21 Jan 2008) | 1 line

  A bunch of header files were not listed as dependencies for object files. Changes to files like Parser/parser.h weren't picked up by make.
........
  r60152 | georg.brandl | 2008-01-21 15:16:46 +0100 (Mon, 21 Jan 2008) | 3 lines

  #1087741: make mmap.mmap the type of mmap objects, not a
  factory function. Allow it to be subclassed.
........
  r60153 | georg.brandl | 2008-01-21 15:18:14 +0100 (Mon, 21 Jan 2008) | 2 lines

  mmap is an extension module.
........
  r60154 | georg.brandl | 2008-01-21 17:28:13 +0100 (Mon, 21 Jan 2008) | 2 lines

  Fix example.
........
  r60155 | georg.brandl | 2008-01-21 17:34:07 +0100 (Mon, 21 Jan 2008) | 2 lines

  #1555501: document plistlib and move it to the general library.
........
  r60156 | georg.brandl | 2008-01-21 17:36:00 +0100 (Mon, 21 Jan 2008) | 2 lines

  Add a stub for bundlebuilder documentation.
........
  r60157 | georg.brandl | 2008-01-21 17:46:58 +0100 (Mon, 21 Jan 2008) | 2 lines

  Removing bundlebuilder docs again -- it's not to be used anymore (see #779825).
........
  r60158 | georg.brandl | 2008-01-21 17:51:51 +0100 (Mon, 21 Jan 2008) | 2 lines

  #997912: acknowledge nested scopes in tutorial.
........
  r60159 | vinay.sajip | 2008-01-21 18:02:26 +0100 (Mon, 21 Jan 2008) | 1 line

  Fix: #1836: Off-by-one bug in TimedRotatingFileHandler rollover calculation. Patch thanks to Kathryn M. Kowalski.
........
  r60161 | georg.brandl | 2008-01-21 18:13:03 +0100 (Mon, 21 Jan 2008) | 2 lines

  Adapt pydoc to new doc URLs.
........
  r60162 | georg.brandl | 2008-01-21 18:17:00 +0100 (Mon, 21 Jan 2008) | 2 lines

  Fix old link.
........
  r60163 | georg.brandl | 2008-01-21 18:22:06 +0100 (Mon, 21 Jan 2008) | 2 lines

  #1726198: replace while 1: fp.readline() with file iteration.
........
  r60164 | georg.brandl | 2008-01-21 18:29:23 +0100 (Mon, 21 Jan 2008) | 2 lines

  Clarify $ behavior in re docstring. #1631394.
........
  r60165 | vinay.sajip | 2008-01-21 18:39:22 +0100 (Mon, 21 Jan 2008) | 1 line

  Minor documentation change - hyperlink tidied up.
........
  r60166 | georg.brandl | 2008-01-21 18:42:40 +0100 (Mon, 21 Jan 2008) | 2 lines

  #1530959: change distutils build dir for --with-pydebug python builds.
........
  r60167 | vinay.sajip | 2008-01-21 19:16:05 +0100 (Mon, 21 Jan 2008) | 1 line

  Updated to include news on recent logging fixes and documentation changes.
........
  r60168 | georg.brandl | 2008-01-21 19:35:49 +0100 (Mon, 21 Jan 2008) | 3 lines

  Issue #1882: when compiling code from a string, encoding cookies in the
  second line of code were not always recognized correctly.
........
  r60170 | georg.brandl | 2008-01-21 19:36:51 +0100 (Mon, 21 Jan 2008) | 2 lines

  Add NEWS entry for #1882.
........
  r60172 | georg.brandl | 2008-01-21 19:41:24 +0100 (Mon, 21 Jan 2008) | 2 lines

  Use original location of document, which has translations.
........
  r60173 | walter.doerwald | 2008-01-21 21:18:04 +0100 (Mon, 21 Jan 2008) | 2 lines

  Follow PEP 8 in module docstring.
........
  r60175 | georg.brandl | 2008-01-21 21:20:53 +0100 (Mon, 21 Jan 2008) | 2 lines

  Adapt to latest doctools refactoring.
........

25 files changed, 382 insertions, 133 deletions

diff --git a/Doc/bugs.rst b/Doc/bugs.rst
index f8f75c2..8b8df81 100644
--- a/Doc/bugs.rst
+++ b/Doc/bugs.rst
@@ -49,7 +49,7 @@ taken on the bug.
 
 .. seealso::
 
-   `How to Report Bugs Effectively <http://www-mice.cs.ucl.ac.uk/multimedia/software/documentation/ReportingBugs.html>`_
+   `How to Report Bugs Effectively <http://www.chiark.greenend.org.uk/~sgtatham/bugs.html>`_
       Article which goes into some detail about how to create a useful bug report.
       This describes what kind of information is useful and why it is useful.
 
diff --git a/Doc/conf.py b/Doc/conf.py
index 5be5d1e..8359674 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -7,23 +7,27 @@
 # The contents of this file are pickled, so don't put values in the namespace
 # that aren't pickleable (module imports are okay, they're removed automatically).
 
+import sys, os, time
+sys.path.append('tools/sphinxext')
+
 # General configuration
 # ---------------------
 
 # General substitutions.
 project = 'Python'
-copyright = '1990-2007, Python Software Foundation'
+copyright = '1990-%s, Python Software Foundation' % time.strftime('%Y')
 
 # The default replacements for |version| and |release|.
-# If '<auto>', Sphinx looks for the Include/patchlevel.h file in the current Python
-# source tree and replaces the values accordingly.
 #
 # The short X.Y version.
 # version = '2.6'
-version = '<auto>'
 # The full version, including alpha/beta/rc tags.
 # release = '2.6a0'
-release = '<auto>'
+
+# We look for the Include/patchlevel.h file in the current Python source tree
+# and replace the values accordingly.
+import patchlevel
+version, release = patchlevel.get_version_info()
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -119,7 +123,6 @@ latex_documents = [
      'What\'s New in Python', 'A. M. Kuchling', 'howto'),
 ]
 # Collect all HOWTOs individually
-import os
 latex_documents.extend(('howto/' + fn, 'howto-' + fn[:-4] + '.tex',
                         'HOWTO', _stdauthor, 'howto')
                        for fn in os.listdir('howto')
diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst
index 5e1dec8..1c7b53f 100644
--- a/Doc/extending/building.rst
+++ b/Doc/extending/building.rst
@@ -80,7 +80,7 @@ example below. ::
           description = 'This is a demo package',
           author = 'Martin v. Loewis',
           author_email = 'martin@v.loewis.de',
-          url = 'http://www.python.org/doc/current/ext/building.html',
+          url = 'http://docs.python.org/extending/building',
           long_description = '''
    This is really just a demo package.
    ''',
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 6adecd7..d6c6b0a 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -335,9 +335,8 @@ Performing Matches
 
 Once you have an object representing a compiled regular expression, what do you
 do with it?  :class:`RegexObject` instances have several methods and attributes.
-Only the most significant ones will be covered here; consult `the Library
-Reference <http://www.python.org/doc/lib/module-re.html>`_ for a complete
-listing.
+Only the most significant ones will be covered here; consult the :mod:`re` docs
+for a complete listing.
 
 +------------------+-----------------------------------------------+
 | Method/Attribute | Purpose                                       |
diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 979f351..28f2e59 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -420,3 +420,5 @@ The function ``opt_move`` below can be used to move options between sections::
            # Create non-existent section
            config.add_section(section2)
            opt_move(config, section1, section2, option)
+       else:
+           config.remove_option(section1, option)
diff --git a/Doc/library/fileformats.rst b/Doc/library/fileformats.rst
index c0c2eed..d2f0639 100644
--- a/Doc/library/fileformats.rst
+++ b/Doc/library/fileformats.rst
@@ -16,3 +16,4 @@ that aren't markup languages or are related to e-mail.
    robotparser.rst
    netrc.rst
    xdrlib.rst
+   plistlib.rst
diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 923d9f2..ce574d0 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -1180,13 +1180,13 @@ also illustrates what dict-like behaviour is needed from an arbitrary
 "dict-like" object for use in the constructor::
 
    import logging
-   
+
    class ConnInfo:
        """
        An example class which shows how an arbitrary class can be used as
        the 'extra' context information repository passed to a LoggerAdapter.
        """
-   
+
        def __getitem__(self, name):
            """
            To allow this instance to look like a dict.
@@ -1199,7 +1199,7 @@ also illustrates what dict-like behaviour is needed from an arbitrary
            else:
                result = self.__dict__.get(name, "?")
            return result
-   
+
        def __iter__(self):
            """
            To allow iteration over keys, which will be merged into
@@ -1208,7 +1208,7 @@ also illustrates what dict-like behaviour is needed from an arbitrary
            keys = ["ip", "user"]
            keys.extend(self.__dict__.keys())
            return keys.__iter__()
-   
+
    if __name__ == "__main__":
        from random import choice
        levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL)
@@ -2133,7 +2133,10 @@ LoggerAdapter Objects
 .. versionadded:: 2.6
 
 :class:`LoggerAdapter` instances are used to conveniently pass contextual
-information into logging calls. For a usage example , see context-info_.
+information into logging calls. For a usage example , see the section on
+`adding contextual information to your logging output`__.
+
+__ context-info_
 
 .. class:: LoggerAdapter(logger, extra)
 
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 37391d0..8ec9885 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -15,7 +15,7 @@ substring by assigning to a slice: ``obj[i1:i2] = '...'``.  You can also read
 and write data starting at the current file position, and :meth:`seek` through
 the file to different positions.
 
-A memory-mapped file is created by the :func:`mmap` function, which is different
+A memory-mapped file is created by the :class:`mmap` constructor, which is different
 on Unix and on Windows.  In either case you must provide a file descriptor for a
 file opened for update. If you wish to map an existing Python file object, use
 its :meth:`fileno` method to obtain the correct value for the *fileno*
@@ -23,7 +23,7 @@ parameter.  Otherwise, you can open the file using the :func:`os.open` function,
 which returns a file descriptor directly (the file still needs to be closed when
 done).
 
-For both the Unix and Windows versions of the function, *access* may be
+For both the Unix and Windows versions of the constructor, *access* may be
 specified as an optional keyword parameter. *access* accepts one of three
 values: :const:`ACCESS_READ`, :const:`ACCESS_WRITE`, or :const:`ACCESS_COPY` to
 specify readonly, write-through or copy-on-write memory respectively. *access*
@@ -37,11 +37,10 @@ not update the underlying file.
 
 To map anonymous memory, -1 should be passed as the fileno along with the length.
 
-
-.. function:: mmap(fileno, length[, tagname[, access[, offset]]])
+.. class:: mmap(fileno, length[, tagname[, access[, offset]]])
 
    **(Windows version)** Maps *length* bytes from the file specified by the file
-   handle *fileno*, and returns a mmap object.  If *length* is larger than the
+   handle *fileno*, and creates a mmap object.  If *length* is larger than the
    current size of the file, the file is extended to contain *length* bytes.  If
    *length* is ``0``, the maximum length of the map is the current size of the
    file, except that if the file is empty Windows raises an exception (you cannot
@@ -59,12 +58,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
    *offset* must be a multiple of the ALLOCATIONGRANULARITY.
 
 
-.. function:: mmap(fileno, length[, flags[, prot[, access[, offset]]]])
+.. class:: mmap(fileno, length[, flags[, prot[, access[, offset]]]])
    :noindex:
 
    **(Unix version)** Maps *length* bytes from the file specified by the file
    descriptor *fileno*, and returns a mmap object.  If *length* is ``0``, the
-   maximum length of the map will be the current size of the file when :func:`mmap`
+   maximum length of the map will be the current size of the file when :class:`mmap`
    is called.
 
    *flags* specifies the nature of the mapping. :const:`MAP_PRIVATE` creates a
@@ -85,7 +84,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
    be relative to the offset from the beginning of the file. *offset* defaults to 0.
    *offset* must be a multiple of the PAGESIZE or ALLOCATIONGRANULARITY.
    
-   This example shows a simple way of using :func:`mmap`::
+   This example shows a simple way of using :class:`mmap`::
 
       import mmap
 
diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
new file mode 100644
index 0000000..508f8e8
--- /dev/null
+++ b/Doc/library/plistlib.rst
@@ -0,0 +1,124 @@
+:mod:`plistlib` --- Generate and parse MacOS X ``.plist`` files
+===============================================================
+
+.. module:: plistlib
+   :synopsis: Generate and parse MacOS X plist files.
+.. moduleauthor:: Jack Jansen
+.. sectionauthor:: Georg Brandl <georg@python.org>
+.. (harvested from docstrings in the original file)
+
+.. versionchanged:: 2.6
+   This module was previously only available in the Mac-specific library, it is
+   now available for all platforms.
+
+.. index::
+   pair: plist; file
+   single: property list
+
+This module provides an interface for reading and writing the "property list"
+XML files used mainly by MacOS X.
+
+The property list (``.plist``) file format is a simple XML pickle supporting
+basic object types, like dictionaries, lists, numbers and strings.  Usually the
+top level object is a dictionary.
+
+Values can be strings, integers, floats, booleans, tuples, lists, dictionaries
+(but only with string keys), :class:`Data` or :class:`datetime.datetime`
+objects.  String values (including dictionary keys) may be unicode strings --
+they will be written out as UTF-8.
+
+The ``<data>`` plist type is supported through the :class:`Data` class.  This is
+a thin wrapper around a Python string.  Use :class:`Data` if your strings
+contain control characters.
+
+.. seealso::
+
+   `PList manual page <http://developer.apple.com/documentation/Darwin/Reference/ManPages/man5/plist.5.html>`
+      Apple's documentation of the file format.
+
+
+This module defines the following functions:
+
+.. function:: readPlist(pathOrFile)
+
+   Read a plist file. *pathOrFile* may either be a file name or a (readable)
+   file object.  Return the unpacked root object (which usually is a
+   dictionary).
+
+   The XML data is parsed using the Expat parser from :mod:`xml.parsers.expat`
+   -- see its documentation for possible exceptions on ill-formed XML.
+   Unknown elements will simply be ignored by the plist parser.
+
+
+.. function:: writePlist(rootObject, pathOrFile)
+
+    Write *rootObject* to a plist file. *pathOrFile* may either be a file name
+    or a (writable) file object.
+
+    A :exc:`TypeError` will be raised if the object is of an unsupported type or
+    a container that contains objects of unsupported types.
+
+
+.. function:: readPlistFromString(data)
+
+   Read a plist from a string.  Return the root object.
+
+
+.. function:: writePlistToString(rootObject)
+
+   Return *rootObject* as a plist-formatted string.
+
+
+
+.. function:: readPlistFromResource(path[, restype='plst'[, resid=0]])
+
+    Read a plist from the resource with type *restype* from the resource fork of
+    *path*.  Availability: MacOS X.
+
+
+.. function:: writePlistToResource(rootObject, path[, restype='plst'[, resid=0]])
+
+    Write *rootObject* as a resource with type *restype* to the resource fork of
+    *path*.  Availability: MacOS X.
+
+
+The following class is available:
+
+.. class:: Data(data)
+
+   Return a "data" wrapper object around the string *data*.  This is used in
+   functions converting from/to plists to represent the ``<data>`` type
+   available in plists.
+
+   It has one attribute, :attr:`data`, that can be used to retrieve the Python
+   string stored in it.
+
+
+Examples
+--------
+
+Generating a plist::
+
+    pl = dict(
+        aString="Doodah",
+        aList=["A", "B", 12, 32.1, [1, 2, 3]],
+        aFloat = 0.1,
+        anInt = 728,
+        aDict=dict(
+            anotherString="<hello & hi there!>",
+            aUnicodeValue=u'M\xe4ssig, Ma\xdf',
+            aTrueValue=True,
+            aFalseValue=False,
+        ),
+        someData = Data("<binary gunk>"),
+        someMoreData = Data("<lots of binary gunk>" * 10),
+        aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())),
+    )
+    # unicode keys are possible, but a little awkward to use:
+    pl[u'\xc5benraa'] = "That was a unicode key."
+    writePlist(pl, fileName)
+
+Parsing a plist::
+
+    pl = readPlist(pathOrFile)
+    print pl["aKey"]
diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst
index 4d74397..41ea1c6 100644
--- a/Doc/library/pydoc.rst
+++ b/Doc/library/pydoc.rst
@@ -57,7 +57,7 @@ documents precisely the version of the module you would get if you started the
 Python interpreter and typed ``import spam``.
 
 Module docs for core modules are assumed to reside in
-http://www.python.org/doc/current/lib/.  This can be overridden by setting the
+http://docs.python.org/library/.  This can be overridden by setting the
 :envvar:`PYTHONDOCS` environment variable to a different URL or to a local
 directory containing the Library Reference Manual pages.
 
diff --git a/Doc/tools/sphinxext/patchlevel.py b/Doc/tools/sphinxext/patchlevel.py
new file mode 100644
index 0000000..971ea61
--- /dev/null
+++ b/Doc/tools/sphinxext/patchlevel.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    patchlevel.py
+    ~~~~~~~~~~~~~
+
+    Extract version info from Include/patchlevel.h.
+    Adapted from Doc/tools/getversioninfo.
+
+    :copyright: 2007-2008 by Georg Brandl.
+    :license: Python license.
+"""
+
+import os
+import re
+import sys
+
+def get_header_version_info(srcdir):
+    patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h')
+
+    # This won't pick out all #defines, but it will pick up the ones we
+    # care about.
+    rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)')
+
+    d = {}
+    f = open(patchlevel_h)
+    try:
+        for line in f:
+            m = rx.match(line)
+            if m is not None:
+                name, value = m.group(1, 2)
+                d[name] = value
+    finally:
+        f.close()
+
+    release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION'])
+    micro = int(d['PY_MICRO_VERSION'])
+    if micro != 0:
+        release += '.' + str(micro)
+
+    level = d['PY_RELEASE_LEVEL']
+    suffixes = {
+        'PY_RELEASE_LEVEL_ALPHA': 'a',
+        'PY_RELEASE_LEVEL_BETA':  'b',
+        'PY_RELEASE_LEVEL_GAMMA': 'c',
+        }
+    if level != 'PY_RELEASE_LEVEL_FINAL':
+        release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL']))
+    return version, release
+
+
+def get_sys_version_info():
+    major, minor, micro, level, serial = sys.version_info
+    release = version = '%s.%s' % (major, minor)
+    if micro:
+        release += '.%s' % micro
+    if level != 'final':
+        release += '%s%s' % (level[0], serial)
+    return version, release
+
+
+def get_version_info():
+    try:
+        return get_header_version_info('.')
+    except (IOError, OSError):
+        version, release = get_sys_version_info()
+        print >>sys.stderr, 'Can\'t get version info from Include/patchlevel.h, ' \
+              'using version of this interpreter (%s).' % release
+        return version, release
diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index 82a8977..3d011d1 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -235,10 +235,11 @@ it.
 The *execution* of a function introduces a new symbol table used for the local
 variables of the function.  More precisely, all variable assignments in a
 function store the value in the local symbol table; whereas variable references
-first look in the local symbol table, then in the global symbol table, and then
-in the table of built-in names. Thus,  global variables cannot be directly
-assigned a value within a function (unless named in a :keyword:`global`
-statement), although they may be referenced.
+first look in the local symbol table, then in the local symbol tables of
+enclosing functions, then in the global symbol table, and finally in the table
+of built-in names. Thus, global variables cannot be directly assigned a value
+within a function (unless named in a :keyword:`global` statement), although they
+may be referenced.
 
 The actual parameters (arguments) to a function call are introduced in the local
 symbol table of the called function when it is called; thus, arguments are
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py
index 1f2ce06..4fe95b0 100644
--- a/Lib/distutils/command/build.py
+++ b/Lib/distutils/command/build.py
@@ -66,6 +66,12 @@ class build(Command):
     def finalize_options(self):
         plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3])
 
+        # Make it so Python 2.x and Python 2.x with --with-pydebug don't
+        # share the same build directories. Doing so confuses the build
+        # process for C modules
+        if hasattr(sys, 'gettotalrefcount'):
+            plat_specifier += '-pydebug'
+
         # 'build_purelib' and 'build_platlib' just default to 'lib' and
         # 'lib.<plat>' under the base build directory.  We only use one of
         # them for a given distribution, though --
diff --git a/Lib/formatter.py b/Lib/formatter.py
index 99b4740..bb8ad20 100644
--- a/Lib/formatter.py
+++ b/Lib/formatter.py
@@ -433,10 +433,7 @@ def test(file = None):
         fp = open(sys.argv[1])
     else:
         fp = sys.stdin
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
+    for line in fp:
         if line == '\n':
             f.end_paragraph(1)
         else:
diff --git a/Lib/keyword.py b/Lib/keyword.py
index 9c49cd2..a7abe2b 100755
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -64,9 +64,7 @@ def main():
     fp = open(iptfile)
     strprog = re.compile('"([^"]+)"')
     lines = []
-    while 1:
-        line = fp.readline()
-        if not line: break
+    for line in fp:
         if '{1, "' in line:
             match = strprog.search(line)
             if match:
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
index b53262e..08bebbd 100644
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -226,13 +226,16 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
             #         Days to rollover is 6 - 5 + 3, or 4.  In this case, it's the
             #         number of days left in the current week (1) plus the number
             #         of days in the next week until the rollover day (3).
+            # The calculations described in 2) and 3) above need to have a day added.
+            # This is because the above time calculation takes us to midnight on this
+            # day, i.e. the start of the next day.
             if when.startswith('W'):
                 day = t[6] # 0 is Monday
                 if day != self.dayOfWeek:
                     if day < self.dayOfWeek:
-                        daysToWait = self.dayOfWeek - day - 1
+                        daysToWait = self.dayOfWeek - day
                     else:
-                        daysToWait = 6 - day + self.dayOfWeek
+                        daysToWait = 6 - day + self.dayOfWeek + 1
                     self.rolloverAt = self.rolloverAt + (daysToWait * (60 * 60 * 24))
 
         #print "Will rollover at %d, %d seconds from now" % (self.rolloverAt, self.rolloverAt - currentTime)
diff --git a/Lib/plat-mac/plistlib.py b/Lib/plistlib.py
index db5eea1..9cd1c8e 100644
--- a/Lib/plat-mac/plistlib.py
+++ b/Lib/plistlib.py
@@ -1,6 +1,6 @@
 """plistlib.py -- a tool to generate and parse MacOSX .plist files.
 
-The PropertList (.plist) file format is a simple XML pickle supporting
+The PropertyList (.plist) file format is a simple XML pickle supporting
 basic object types, like dictionaries, lists, numbers and strings.
 Usually the top level object is a dictionary.
 
@@ -12,8 +12,8 @@ To parse a plist from a file, use the readPlist(pathOrFile) function,
 with a file name or a (readable) file object as the only argument. It
 returns the top level object (again, usually a dictionary).
 
-To work with plist data in bytes objects, you can use readPlistFromBytes()
-and writePlistToBytes().
+To work with plist data in strings, you can use readPlistFromString()
+and writePlistToString().
 
 Values can be strings, integers, floats, booleans, tuples, lists,
 dictionaries, Data or datetime.datetime objects. String values (including
@@ -21,24 +21,24 @@ dictionary keys) may be unicode strings -- they will be written out as
 UTF-8.
 
 The <data> plist type is supported through the Data class. This is a
-thin wrapper around a Python bytes object.
+thin wrapper around a Python string.
 
 Generate Plist example:
 
     pl = dict(
         aString="Doodah",
         aList=["A", "B", 12, 32.1, [1, 2, 3]],
-        aFloat = 0.1,
-        anInt = 728,
+        aFloat=0.1,
+        anInt=728,
         aDict=dict(
             anotherString="<hello & hi there!>",
             aUnicodeValue=u'M\xe4ssig, Ma\xdf',
             aTrueValue=True,
             aFalseValue=False,
         ),
-        someData = Data(b"<binary gunk>"),
-        someMoreData = Data(b"<lots of binary gunk>" * 10),
-        aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())),
+        someData=Data("<binary gunk>"),
+        someMoreData=Data("<lots of binary gunk>" * 10),
+        aDate=datetime.datetime.fromtimestamp(time.mktime(time.gmtime())),
     )
     # unicode keys are possible, but a little awkward to use:
     pl[u'\xc5benraa'] = "That was a unicode key."
@@ -52,7 +52,7 @@ Parse Plist example:
 
 
 __all__ = [
-    "readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes",
+    "readPlist", "writePlist", "readPlistFromString", "writePlistToString",
     "readPlistFromResource", "writePlistToResource",
     "Plist", "Data", "Dict"
 ]
@@ -60,7 +60,7 @@ __all__ = [
 
 import binascii
 import datetime
-from io import BytesIO
+from io import StringIO
 import re
 
 
@@ -69,10 +69,10 @@ def readPlist(pathOrFile):
     (readable) file object. Return the unpacked root object (which
     usually is a dictionary).
     """
-    didOpen = False
+    didOpen = 0
     if isinstance(pathOrFile, str):
-        pathOrFile = open(pathOrFile, 'rb')
-        didOpen = True
+        pathOrFile = open(pathOrFile)
+        didOpen = 1
     p = PlistParser()
     rootObject = p.parse(pathOrFile)
     if didOpen:
@@ -84,10 +84,10 @@ def writePlist(rootObject, pathOrFile):
     """Write 'rootObject' to a .plist file. 'pathOrFile' may either be a
     file name or a (writable) file object.
     """
-    didOpen = False
+    didOpen = 0
     if isinstance(pathOrFile, str):
-        pathOrFile = open(pathOrFile, 'wb')
-        didOpen = True
+        pathOrFile = open(pathOrFile, "w")
+        didOpen = 1
     writer = PlistWriter(pathOrFile)
     writer.writeln("<plist version=\"1.0\">")
     writer.writeValue(rootObject)
@@ -96,16 +96,16 @@ def writePlist(rootObject, pathOrFile):
         pathOrFile.close()
 
 
-def readPlistFromBytes(data):
-    """Read a plist data from a bytes object. Return the root object.
+def readPlistFromString(data):
+    """Read a plist data from a string. Return the root object.
     """
-    return readPlist(BytesIO(data))
+    return readPlist(StringIO(data))
 
 
-def writePlistToBytes(rootObject):
-    """Return 'rootObject' as a plist-formatted bytes object.
+def writePlistToString(rootObject):
+    """Return 'rootObject' as a plist-formatted string.
     """
-    f = BytesIO()
+    f = StringIO()
     writePlist(rootObject, f)
     return f.getvalue()
 
@@ -145,6 +145,7 @@ def writePlistToResource(rootObject, path, restype='plst', resid=0):
 
 
 class DumbXMLWriter:
+
     def __init__(self, file, indentLevel=0, indent="\t"):
         self.file = file
         self.stack = []
@@ -164,19 +165,16 @@ class DumbXMLWriter:
 
     def simpleElement(self, element, value=None):
         if value is not None:
-            value = _escape(value)
+            value = _escapeAndEncode(value)
             self.writeln("<%s>%s</%s>" % (element, value, element))
         else:
             self.writeln("<%s/>" % element)
 
     def writeln(self, line):
         if line:
-            # plist has fixed encoding of utf-8
-            if isinstance(line, str):
-                line = line.encode('utf-8')
-            self.file.write(self.indentLevel * self.indent)
-            self.file.write(line)
-        self.file.write(b'\n')
+            self.file.write(self.indentLevel * self.indent + line + "\n")
+        else:
+            self.file.write("\n")
 
 
 # Contents should conform to a subset of ISO 8601
@@ -207,7 +205,7 @@ _controlCharPat = re.compile(
     r"[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f"
     r"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]")
 
-def _escape(text):
+def _escapeAndEncode(text):
     m = _controlCharPat.search(text)
     if m is not None:
         raise ValueError("strings can't contains control characters; "
@@ -217,17 +215,17 @@ def _escape(text):
     text = text.replace("&", "&amp;")       # escape '&'
     text = text.replace("<", "&lt;")        # escape '<'
     text = text.replace(">", "&gt;")        # escape '>'
-    return text
+    return text.encode("utf-8")             # encode as UTF-8
 
 
-PLISTHEADER = b"""\
+PLISTHEADER = """\
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 """
 
 class PlistWriter(DumbXMLWriter):
 
-    def __init__(self, file, indentLevel=0, indent=b"\t", writeHeader=1):
+    def __init__(self, file, indentLevel=0, indent="\t", writeHeader=1):
         if writeHeader:
             file.write(PLISTHEADER)
         DumbXMLWriter.__init__(self, file, indentLevel, indent)
@@ -260,9 +258,9 @@ class PlistWriter(DumbXMLWriter):
     def writeData(self, data):
         self.beginElement("data")
         self.indentLevel -= 1
-        maxlinelength = 76 - len(self.indent.replace(b"\t", b" " * 8) *
+        maxlinelength = 76 - len(self.indent.replace("\t", " " * 8) *
                                  self.indentLevel)
-        for line in data.asBase64(maxlinelength).split(b"\n"):
+        for line in data.asBase64(maxlinelength).split("\n"):
             if line:
                 self.writeln(line)
         self.indentLevel += 1
@@ -270,7 +268,8 @@ class PlistWriter(DumbXMLWriter):
 
     def writeDict(self, d):
         self.beginElement("dict")
-        items = sorted(d.items())
+        items = list(d.items())
+        items.sort()
         for key, value in items:
             if not isinstance(key, str):
                 raise TypeError("keys must be strings")
@@ -322,7 +321,7 @@ class Dict(_InternalDict):
         from warnings import warn
         warn("The plistlib.Dict class is deprecated, use builtin dict instead",
              PendingDeprecationWarning)
-        super().__init__(**kwargs)
+        super(Dict, self).__init__(**kwargs)
 
 
 class Plist(_InternalDict):
@@ -335,7 +334,7 @@ class Plist(_InternalDict):
         from warnings import warn
         warn("The Plist class is deprecated, use the readPlist() and "
              "writePlist() functions instead", PendingDeprecationWarning)
-        super().__init__(**kwargs)
+        super(Plist, self).__init__(**kwargs)
 
     def fromFile(cls, pathOrFile):
         """Deprecated. Use the readPlist() function instead."""
@@ -357,33 +356,31 @@ def _encodeBase64(s, maxlinelength=76):
     for i in range(0, len(s), maxbinsize):
         chunk = s[i : i + maxbinsize]
         pieces.append(binascii.b2a_base64(chunk))
-    return b''.join(pieces)
+    return "".join(pieces)
 
 class Data:
 
     """Wrapper for binary data."""
 
     def __init__(self, data):
-        if not isinstance(data, bytes):
-            raise TypeError("data must be as bytes")
         self.data = data
 
-    @classmethod
     def fromBase64(cls, data):
         # base64.decodestring just calls binascii.a2b_base64;
         # it seems overkill to use both base64 and binascii.
         return cls(binascii.a2b_base64(data))
+    fromBase64 = classmethod(fromBase64)
 
     def asBase64(self, maxlinelength=76):
         return _encodeBase64(self.data, maxlinelength)
 
-    def __eq__(self, other):
+    def __cmp__(self, other):
         if isinstance(other, self.__class__):
-            return self.data == other.data
+            return cmp(self.data, other.data)
         elif isinstance(other, str):
-            return self.data == other
+            return cmp(self.data, other)
         else:
-            return id(self) == id(other)
+            return cmp(id(self), id(other))
 
     def __repr__(self):
         return "%s(%s)" % (self.__class__.__name__, repr(self.data))
@@ -430,7 +427,11 @@ class PlistParser:
             self.stack[-1].append(value)
 
     def getData(self):
-        data = ''.join(self.data)
+        data = "".join(self.data)
+        try:
+            data = data.encode("ascii")
+        except UnicodeError:
+            pass
         self.data = []
         return data
 
@@ -464,6 +465,6 @@ class PlistParser:
     def end_string(self):
         self.addObject(self.getData())
     def end_data(self):
-        self.addObject(Data.fromBase64(self.getData().encode("utf-8")))
+        self.addObject(Data.fromBase64(self.getData()))
     def end_date(self):
         self.addObject(_dateFromString(self.getData()))
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 2a1e98f..d5fb91b 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -27,7 +27,7 @@ to a file named "<name>.html".
 
 Module docs for core modules are assumed to be in
 
-    http://www.python.org/doc/current/lib/
+    http://docs.python.org/library/
 
 This can be overridden by setting the PYTHONDOCS environment variable
 to a different URL or to a local directory containing the Library
@@ -341,7 +341,7 @@ class Doc:
             file = '(built-in)'
 
         docloc = os.environ.get("PYTHONDOCS",
-                                "http://www.python.org/doc/current/lib")
+                                "http://docs.python.org/library")
         basedir = os.path.join(sys.exec_prefix, "lib",
                                "python"+sys.version[0:3])
         if (isinstance(object, type(os)) and
@@ -350,11 +350,10 @@ class Doc:
                                  'thread', 'zipimport') or
              (file.startswith(basedir) and
               not file.startswith(os.path.join(basedir, 'site-packages'))))):
-            htmlfile = "module-%s.html" % object.__name__
             if docloc.startswith("http://"):
-                docloc = "%s/%s" % (docloc.rstrip("/"), htmlfile)
+                docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__)
             else:
-                docloc = os.path.join(docloc, htmlfile)
+                docloc = os.path.join(docloc, object.__name__ + ".html")
         else:
             docloc = None
         return docloc
@@ -537,7 +536,7 @@ class HTMLDoc(Doc):
                 url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
                 results.append('<a href="%s">%s</a>' % (url, escape(all)))
             elif pep:
-                url = 'http://www.python.org/peps/pep-%04d.html' % int(pep)
+                url = 'http://www.python.org/peps/pep-%04d' % int(pep)
                 results.append('<a href="%s">%s</a>' % (url, escape(all)))
             elif text[end:end+1] == '(':
                 results.append(self.namelink(name, methods, funcs, classes))
@@ -1729,7 +1728,7 @@ has the same effect as typing a particular string at the help> prompt.
 Welcome to Python %s!  This is the online help utility.
 
 If this is your first time using Python, you should definitely check out
-the tutorial on the Internet at http://www.python.org/doc/tut/.
+the tutorial on the Internet at http://docs.python.org/tutorial/.
 
 Enter the name of any module, keyword, or topic to get help on writing
 Python programs and using Python modules.  To quit this help utility and
diff --git a/Lib/re.py b/Lib/re.py
index 1e0c468..91b119c 100644
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -29,7 +29,8 @@ concatenate ordinary characters, so last matches the string 'last'.
 The special characters are:
     "."      Matches any character except a newline.
     "^"      Matches the start of the string.
-    "$"      Matches the end of the string.
+    "$"      Matches the end of the string or just before the newline at
+             the end of the string.
     "*"      Matches 0 or more (greedy) repetitions of the preceding RE.
              Greedy means that it will match as many repetitions as possible.
     "+"      Matches 1 or more (greedy) repetitions of the preceding RE.
@@ -83,8 +84,10 @@ This module exports the following functions:
 Some of the functions in this module takes flags as optional parameters:
     I  IGNORECASE  Perform case-insensitive matching.
     L  LOCALE      Make \w, \W, \b, \B, dependent on the current locale.
-    M  MULTILINE   "^" matches the beginning of lines as well as the string.
-                   "$" matches the end of lines as well as the string.
+    M  MULTILINE   "^" matches the beginning of lines (after a newline)
+                   as well as the string.
+                   "$" matches the end of lines (before a newline) as well
+                   as the end of the string.
     S  DOTALL      "." matches any character at all, including the newline.
     X  VERBOSE     Ignore whitespace and comments for nicer looking RE's.
     U  UNICODE     Make \w, \W, \b, \B, dependent on the Unicode locale.
diff --git a/Lib/test/test_pep263.py b/Lib/test/test_pep263.py
index dd13558..cacf1d6 100644
--- a/Lib/test/test_pep263.py
+++ b/Lib/test/test_pep263.py
@@ -16,6 +16,14 @@ class PEP263Test(unittest.TestCase):
             b'\\\xd0\x9f'

         )

 

+    def test_compilestring(self):

+        # see #1882

+        c = compile("\n# coding: utf-8\nu = '\xc3\xb3'\n", "dummy", "exec")

+        d = {}

+        exec(c, d)

+        self.assertEqual(d['u'], '\xf3')

+

+

 def test_main():

     test_support.run_unittest(PEP263Test)

 

diff --git a/Lib/urlparse.py b/Lib/urlparse.py
index 0c05dfe..30de699 100644
--- a/Lib/urlparse.py
+++ b/Lib/urlparse.py
@@ -305,9 +305,7 @@ def test():
     else:
         from io import StringIO
         fp = StringIO(test_input)
-    while 1:
-        line = fp.readline()
-        if not line: break
+    for line in fp:
         words = line.split()
         if not words:
             continue
diff --git a/Makefile.pre.in b/Makefile.pre.in
index d762185..406478c 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -223,6 +223,10 @@ PGOBJS=		\
 		Parser/printgrammar.o \
 		Parser/pgenmain.o
 
+PARSER_HEADERS= \
+		Parser/parser.h \
+		Parser/tokenizer.h
+
 PGENOBJS=	$(PGENMAIN) $(POBJS) $(PGOBJS)
 
 ##########################################################################
@@ -542,13 +546,16 @@ Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \
 # Header files
 
 PYTHON_HEADERS= \
-		Include/Python.h \
 		Include/Python-ast.h \
-		Include/asdl.h \
+		Include/Python.h \
 		Include/abstract.h \
+		Include/asdl.h \
+		Include/ast.h \
+		Include/bitset.h \
 		Include/boolobject.h \
 		Include/bytes_methods.h \
 		Include/bytesobject.h \
+		Include/cellobject.h \
 		Include/ceval.h \
 		Include/classobject.h \
 		Include/cobject.h \
@@ -559,48 +566,62 @@ PYTHON_HEADERS= \
 		Include/descrobject.h \
 		Include/dictobject.h \
 		Include/enumobject.h \
-		Include/genobject.h \
+		Include/errcode.h \
+		Include/eval.h \
 		Include/fileobject.h \
 		Include/floatobject.h \
 		Include/formatter_unicode.h \
+		Include/frameobject.h \
 		Include/funcobject.h \
+		Include/genobject.h \
 		Include/import.h \
 		Include/intrcheck.h \
 		Include/iterobject.h \
 		Include/listobject.h \
 		Include/longintrepr.h \
 		Include/longobject.h \
+		Include/marshal.h \
 		Include/memoryobject.h \
+		Include/metagrammar.h \
 		Include/methodobject.h \
 		Include/modsupport.h \
 		Include/moduleobject.h \
+		Include/node.h \
 		Include/object.h \
 		Include/objimpl.h \
+		Include/opcode.h \
+		Include/osdefs.h \
 		Include/parsetok.h \
 		Include/patchlevel.h \
+		Include/pgen.h \
+		Include/pgenheaders.h \
 		Include/pyarena.h \
 		Include/pydebug.h \
 		Include/pyerrors.h \
 		Include/pyfpe.h \
+		Include/pygetopt.h \
 		Include/pymem.h \
 		Include/pyport.h \
 		Include/pystate.h \
-		Include/pystrtod.h \
 		Include/pystrcmp.h \
+		Include/pystrtod.h \
 		Include/pythonrun.h \
+		Include/pythread.h \
 		Include/rangeobject.h \
-                Include/setobject.h \
+		Include/setobject.h \
 		Include/sliceobject.h \
 		Include/stringobject.h \
-		Include/structseq.h \
 		Include/structmember.h \
+		Include/structseq.h \
 		Include/symtable.h \
 		Include/sysmodule.h \
 		Include/traceback.h \
 		Include/tupleobject.h \
+		Include/ucnhash.h \
 		Include/unicodeobject.h \
 		Include/weakrefobject.h \
-		pyconfig.h
+		pyconfig.h \
+		$(PARSER_HEADERS)
 
 $(LIBRARY_OBJS) $(MODOBJS) Modules/python.o: $(PYTHON_HEADERS)
 
diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index e06cefb..ac39a23 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -895,6 +895,9 @@ static PyBufferProcs mmap_as_buffer = {
         (releasebufferproc)mmap_buffer_releasebuf,
 };
 
+static PyObject *
+new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict);
+
 PyDoc_STRVAR(mmap_doc,
 "Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\
 \n\
@@ -920,7 +923,7 @@ To map anonymous memory, pass -1 as the fileno (both versions).");
 
 
 static PyTypeObject mmap_object_type = {
-	PyVarObject_HEAD_INIT(0, 0) /* patched in module init */
+	PyVarObject_HEAD_INIT(NULL, 0)
 	"mmap.mmap",				/* tp_name */
 	sizeof(mmap_object),			/* tp_size */
 	0,					/* tp_itemsize */
@@ -940,16 +943,26 @@ static PyTypeObject mmap_object_type = {
 	PyObject_GenericGetAttr,		/*tp_getattro*/
 	0,					/*tp_setattro*/
 	&mmap_as_buffer,			/*tp_as_buffer*/
-	Py_TPFLAGS_DEFAULT,			/*tp_flags*/
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,		/*tp_flags*/
 	mmap_doc,				/*tp_doc*/
 	0,					/* tp_traverse */
 	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
-	0,					/* tp_iter */
-	0,					/* tp_iternext */
+	0,		                        /* tp_iter */
+	0,		                        /* tp_iternext */
 	mmap_object_methods,			/* tp_methods */
-
+	0,					/* tp_members */
+	0,					/* tp_getset */
+	0,					/* tp_base */
+	0,					/* tp_dict */
+	0,					/* tp_descr_get */
+	0,					/* tp_descr_set */
+	0,					/* tp_dictoffset */
+	0,                                      /* tp_init */
+	PyType_GenericAlloc,			/* tp_alloc */
+	new_mmap_object,			/* tp_new */
+	PyObject_Del,                           /* tp_free */
 };
 
 
@@ -981,7 +994,7 @@ _GetMapSize(PyObject *o, const char* param)
 
 #ifdef UNIX
 static PyObject *
-new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
+new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
 {
 #ifdef HAVE_FSTAT
 	struct stat st;
@@ -1049,7 +1062,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 		}
 	}
 #endif
-	m_obj = PyObject_New(mmap_object, &mmap_object_type);
+	m_obj = (mmap_object *)type->tp_alloc(type, 0);
 	if (m_obj == NULL) {return NULL;}
 	m_obj->data = NULL;
 	m_obj->size = (size_t) map_size;
@@ -1104,7 +1117,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 
 #ifdef MS_WINDOWS
 static PyObject *
-new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
+new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
 {
 	mmap_object *m_obj;
 	PyObject *map_size_obj = NULL, *offset_obj = NULL;
@@ -1173,7 +1186,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 		lseek(fileno, 0, SEEK_SET);
 	}
 
-	m_obj = PyObject_New(mmap_object, &mmap_object_type);
+	m_obj = (mmap_object *)type->tp_alloc(type, 0);
 	if (m_obj == NULL)
 		return NULL;
 	/* Set every field to an invalid marker, so we can safely
@@ -1288,13 +1301,6 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 }
 #endif /* MS_WINDOWS */
 
-/* List of functions exported by this module */
-static struct PyMethodDef mmap_functions[] = {
-	{"mmap",	(PyCFunction) new_mmap_object,
-	 METH_VARARGS|METH_KEYWORDS, mmap_doc},
-	{NULL,		NULL}	     /* Sentinel */
-};
-
 static void
 setint(PyObject *d, const char *name, long value)
 {
@@ -1305,14 +1311,14 @@ setint(PyObject *d, const char *name, long value)
 }
 
 PyMODINIT_FUNC
-	initmmap(void)
+initmmap(void)
 {
 	PyObject *dict, *module;
 
-	/* Patch the object type */
-	Py_TYPE(&mmap_object_type) = &PyType_Type;
+	if (PyType_Ready(&mmap_object_type) < 0)
+		return;
 
-	module = Py_InitModule("mmap", mmap_functions);
+	module = Py_InitModule("mmap", NULL);
 	if (module == NULL)
 		return;
 	dict = PyModule_GetDict(module);
@@ -1320,6 +1326,7 @@ PyMODINIT_FUNC
 		return;
 	mmap_module_error = PyExc_EnvironmentError;
 	PyDict_SetItemString(dict, "error", mmap_module_error);
+	PyDict_SetItemString(dict, "mmap", (PyObject*) &mmap_object_type);
 #ifdef PROT_EXEC
 	setint(dict, "PROT_EXEC", PROT_EXEC);
 #endif
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 3733f49..36ea466 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -640,6 +640,7 @@ decode_str(const char *str, struct tok_state *tok)
 {
 	PyObject* utf8 = NULL;
 	const char *s;
+	char *newl[2] = {NULL, NULL};
 	int lineno = 0;
 	tok->enc = NULL;
 	tok->str = str;
@@ -656,13 +657,23 @@ decode_str(const char *str, struct tok_state *tok)
 	for (s = str;; s++) {
 		if (*s == '\0') break;
 		else if (*s == '\n') {
+			newl[lineno] = s;
 			lineno++;
 			if (lineno == 2) break;
 		}
 	}
 	tok->enc = NULL;
-	if (!check_coding_spec(str, s - str, tok, buf_setreadl))
-		return error_ret(tok);
+	/* need to check line 1 and 2 separately since check_coding_spec
+	   assumes a single line as input */
+	if (newl[0]) {
+		if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
+			return error_ret(tok);
+		if (tok->enc == NULL && newl[1]) {
+			if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
+					       tok, buf_setreadl))
+				return error_ret(tok);
+		}
+	}
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);
 		utf8 = translate_into_utf8(str, tok->enc);
diff --git a/Tools/pynche/ColorDB.py b/Tools/pynche/ColorDB.py
index 2e7a1b5..3b978fb 100644
--- a/Tools/pynche/ColorDB.py
+++ b/Tools/pynche/ColorDB.py
@@ -50,10 +50,7 @@ class ColorDB:
         self.__byname = {}
         # all unique names (non-aliases).  built-on demand
         self.__allnames = None
-        while 1:
-            line = fp.readline()
-            if not line:
-                break
+        for line in fp:
             # get this compiled regular expression from derived class
             mo = self._re.match(line)
             if not mo: