summaryrefslogtreecommitdiffstats
path: root/Lib/pyclbr.py
blob: d7dba9761851ea8ff4439d557cf704c7f701b501 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
"""Parse a Python module and describe its classes and methods.

Parse enough of a Python file to recognize imports and class and
method definitions, and to find out the superclasses of a class.

The interface consists of a single function:
        readmodule_ex(module [, path])
where module is the name of a Python module, and path is an optional
list of directories where the module is to be searched.  If present,
path is prepended to the system search path sys.path.  The return
value is a dictionary.  The keys of the dictionary are the names of
the classes defined in the module (including classes that are defined
via the from XXX import YYY construct).  The values are class
instances of the class Class defined here.  One special key/value pair
is present for packages: the key '__path__' has a list as its value
which contains the package search path.

A class is described by the class Class in this module.  Instances
of this class have the following instance variables:
        module -- the module name
        name -- the name of the class
        super -- a list of super classes (Class instances)
        methods -- a dictionary of methods
        file -- the file in which the class was defined
        lineno -- the line in the file on which the class statement occurred
The dictionary of methods uses the method names as keys and the line
numbers on which the method was defined as values.
If the name of a super class is not recognized, the corresponding
entry in the list of super classes is not a class instance but a
string giving the name of the super class.  Since import statements
are recognized and imported modules are scanned as well, this
shouldn't happen often.

A function is described by the class Function in this module.
Instances of this class have the following instance variables:
        module -- the module name
        name -- the name of the class
        file -- the file in which the class was defined
        lineno -- the line in the file on which the class statement occurred
"""

import io
import sys
import importlib.util
import tokenize
from token import NAME, DEDENT, OP

__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]

_modules = {}                           # cache of modules we've seen

# each Python class is represented by an instance of this class
class Class:
    '''Class to represent a Python class.'''
    def __init__(self, module, name, super, file, lineno):
        self.module = module
        self.name = name
        if super is None:
            super = []
        self.super = super
        self.methods = {}
        self.file = file
        self.lineno = lineno

    def _addmethod(self, name, lineno):
        self.methods[name] = lineno

class Function:
    '''Class to represent a top-level Python function'''
    def __init__(self, module, name, file, lineno):
        self.module = module
        self.name = name
        self.file = file
        self.lineno = lineno

def readmodule(module, path=None):
    '''Backwards compatible interface.

    Call readmodule_ex() and then only keep Class objects from the
    resulting dictionary.'''

    res = {}
    for key, value in _readmodule(module, path or []).items():
        if isinstance(value, Class):
            res[key] = value
    return res

def readmodule_ex(module, path=None):
    '''Read a module file and return a dictionary of classes.

    Search for MODULE in PATH and sys.path, read and parse the
    module and return a dictionary with one entry for each class
    found in the module.
    '''
    return _readmodule(module, path or [])

def _readmodule(module, path, inpackage=None):
    '''Do the hard work for readmodule[_ex].

    If INPACKAGE is given, it must be the dotted name of the package in
    which we are searching for a submodule, and then PATH must be the
    package search path; otherwise, we are searching for a top-level
    module, and PATH is combined with sys.path.
    '''
    # Compute the full module name (prepending inpackage if set)
    if inpackage is not None:
        fullmodule = "%s.%s" % (inpackage, module)
    else:
        fullmodule = module

    # Check in the cache
    if fullmodule in _modules:
        return _modules[fullmodule]

    # Initialize the dict for this module's contents
    dict = {}

    # Check if it is a built-in module; we don't do much for these
    if module in sys.builtin_module_names and inpackage is None:
        _modules[module] = dict
        return dict

    # Check for a dotted module name
    i = module.rfind('.')
    if i >= 0:
        package = module[:i]
        submodule = module[i+1:]
        parent = _readmodule(package, path, inpackage)
        if inpackage is not None:
            package = "%s.%s" % (inpackage, package)
        if not '__path__' in parent:
            raise ImportError('No package named {}'.format(package))
        return _readmodule(submodule, parent['__path__'], package)

    # Search the path for the module
    f = None
    if inpackage is not None:
        search_path = path
    else:
        search_path = path + sys.path
    # XXX This will change once issue19944 lands.
    spec = importlib.util._find_spec_from_path(fullmodule, search_path)
    _modules[fullmodule] = dict
    # is module a package?
    if spec.submodule_search_locations is not None:
        dict['__path__'] = spec.submodule_search_locations
    try:
        source = spec.loader.get_source(fullmodule)
        if source is None:
            return dict
    except (AttributeError, ImportError):
        # not Python source, can't do anything with this module
        return dict

    fname = spec.loader.get_filename(fullmodule)

    f = io.StringIO(source)

    stack = [] # stack of (class, indent) pairs

    g = tokenize.generate_tokens(f.readline)
    try:
        for tokentype, token, start, _end, _line in g:
            if tokentype == DEDENT:
                lineno, thisindent = start
                # close nested classes and defs
                while stack and stack[-1][1] >= thisindent:
                    del stack[-1]
            elif token == 'def':
                lineno, thisindent = start
                # close previous nested classes and defs
                while stack and stack[-1][1] >= thisindent:
                    del stack[-1]
                tokentype, meth_name, start = next(g)[0:3]
                if tokentype != NAME:
                    continue # Syntax error
                if stack:
                    cur_class = stack[-1][0]
                    if isinstance(cur_class, Class):
                        # it's a method
                        cur_class._addmethod(meth_name, lineno)
                    # else it's a nested def
                else:
                    # it's a function
                    dict[meth_name] = Function(fullmodule, meth_name,
                                               fname, lineno)
                stack.append((None, thisindent)) # Marker for nested fns
            elif token == 'class':
                lineno, thisindent = start
                # close previous nested classes and defs
                while stack and stack[-1][1] >= thisindent:
                    del stack[-1]
                tokentype, class_name, start = next(g)[0:3]
                if tokentype != NAME:
                    continue # Syntax error
                # parse what follows the class name
                tokentype, token, start = next(g)[0:3]
                inherit = None
                if token == '(':
                    names = [] # List of superclasses
                    # there's a list of superclasses
                    level = 1
                    super = [] # Tokens making up current superclass
                    while True:
                        tokentype, token, start = next(g)[0:3]
                        if token in (')', ',') and level == 1:
                            n = "".join(super)
                            if n in dict:
                                # we know this super class
                                n = dict[n]
                            else:
                                c = n.split('.')
                                if len(c) > 1:
                                    # super class is of the form
                                    # module.class: look in module for
                                    # class
                                    m = c[-2]
                                    c = c[-1]
                                    if m in _modules:
                                        d = _modules[m]
                                        if c in d:
                                            n = d[c]
                            names.append(n)
                            super = []
                        if token == '(':
                            level += 1
                        elif token == ')':
                            level -= 1
                            if level == 0:
                                break
                        elif token == ',' and level == 1:
                            pass
                        # only use NAME and OP (== dot) tokens for type name
                        elif tokentype in (NAME, OP) and level == 1:
                            super.append(token)
                        # expressions in the base list are not supported
                    inherit = names
                cur_class = Class(fullmodule, class_name, inherit,
                                  fname, lineno)
                if not stack:
                    dict[class_name] = cur_class
                stack.append((cur_class, thisindent))
            elif token == 'import' and start[1] == 0:
                modules = _getnamelist(g)
                for mod, _mod2 in modules:
                    try:
                        # Recursively read the imported module
                        if inpackage is None:
                            _readmodule(mod, path)
                        else:
                            try:
                                _readmodule(mod, path, inpackage)
                            except ImportError:
                                _readmodule(mod, [])
                    except:
                        # If we can't find or parse the imported module,
                        # too bad -- don't die here.
                        pass
            elif token == 'from' and start[1] == 0:
                mod, token = _getname(g)
                if not mod or token != "import":
                    continue
                names = _getnamelist(g)
                try:
                    # Recursively read the imported module
                    d = _readmodule(mod, path, inpackage)
                except:
                    # If we can't find or parse the imported module,
                    # too bad -- don't die here.
                    continue
                # add any classes that were defined in the imported module
                # to our name space if they were mentioned in the list
                for n, n2 in names:
                    if n in d:
                        dict[n2 or n] = d[n]
                    elif n == '*':
                        # don't add names that start with _
                        for n in d:
                            if n[0] != '_':
                                dict[n] = d[n]
    except StopIteration:
        pass

    f.close()
    return dict

def _getnamelist(g):
    # Helper to get a comma-separated list of dotted names plus 'as'
    # clauses.  Return a list of pairs (name, name2) where name2 is
    # the 'as' name, or None if there is no 'as' clause.
    names = []
    while True:
        name, token = _getname(g)
        if not name:
            break
        if token == 'as':
            name2, token = _getname(g)
        else:
            name2 = None
        names.append((name, name2))
        while token != "," and "\n" not in token:
            token = next(g)[1]
        if token != ",":
            break
    return names

def _getname(g):
    # Helper to get a dotted name, return a pair (name, token) where
    # name is the dotted name, or None if there was no dotted name,
    # and token is the next input token.
    parts = []
    tokentype, token = next(g)[0:2]
    if tokentype != NAME and token != '*':
        return (None, token)
    parts.append(token)
    while True:
        tokentype, token = next(g)[0:2]
        if token != '.':
            break
        tokentype, token = next(g)[0:2]
        if tokentype != NAME:
            break
        parts.append(token)
    return (".".join(parts), token)

def _main():
    # Main program for testing.
    import os
    from operator import itemgetter
    mod = sys.argv[1]
    if os.path.exists(mod):
        path = [os.path.dirname(mod)]
        mod = os.path.basename(mod)
        if mod.lower().endswith(".py"):
            mod = mod[:-3]
    else:
        path = []
    dict = readmodule_ex(mod, path)
    objs = list(dict.values())
    objs.sort(key=lambda a: getattr(a, 'lineno', 0))
    for obj in objs:
        if isinstance(obj, Class):
            print("class", obj.name, obj.super, obj.lineno)
            methods = sorted(obj.methods.items(), key=itemgetter(1))
            for name, lineno in methods:
                if name != "__path__":
                    print("  def", name, lineno)
        elif isinstance(obj, Function):
            print("def", obj.name, obj.lineno)

if __name__ == "__main__":
    _main()