Parser/tokenizer/tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14

#ifndef Py_TOKENIZER_H
#define Py_TOKENIZER_H

#include "Python.h"

struct tok_state *_PyTokenizer_FromString(const char *, int, int);
struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int);
struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
                                              const char *, const char *);

#define tok_dump _Py_tok_dump

#endif /* !Py_TOKENIZER_H */