1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
n1278' href='#n1278'>1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
|
/* csv module */
/*
This module provides the low-level underpinnings of a CSV reading/writing
module. Users should not use this module directly, but import the csv.py
module instead.
*/
#define MODULE_VERSION "1.0"
#include "Python.h"
#include "structmember.h"
#define IS_BASESTRING(o) \
PyUnicode_Check(o)
static PyObject *error_obj; /* CSV exception */
static PyObject *dialects; /* Dialect registry */
static long field_limit = 128 * 1024; /* max parsed field size */
typedef enum {
START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD,
IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,
EAT_CRNL
} ParserState;
typedef enum {
QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE
} QuoteStyle;
typedef struct {
QuoteStyle style;
char *name;
} StyleDesc;
static StyleDesc quote_styles[] = {
{ QUOTE_MINIMAL, "QUOTE_MINIMAL" },
{ QUOTE_ALL, "QUOTE_ALL" },
{ QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" },
{ QUOTE_NONE, "QUOTE_NONE" },
{ 0 }
};
typedef struct {
PyObject_HEAD
int doublequote; /* is " represented by ""? */
Py_UNICODE delimiter; /* field separator */
Py_UNICODE quotechar; /* quote character */
Py_UNICODE escapechar; /* escape character */
int skipinitialspace; /* ignore spaces following delimiter? */
PyObject *lineterminator; /* string to write between records */
int quoting; /* style of quoting to write */
int strict; /* raise exception on bad CSV */
} DialectObj;
static PyTypeObject Dialect_Type;
typedef struct {
PyObject_HEAD
PyObject *input_iter; /* iterate over this for input lines */
DialectObj *dialect; /* parsing dialect */
PyObject *fields; /* field list for current record */
ParserState state; /* current CSV parse state */
Py_UNICODE *field; /* build current field in here */
Py_ssize_t field_size; /* size of allocated buffer */
Py_ssize_t field_len; /* length of current field */
int numeric_field; /* treat field as numeric */
unsigned long line_num; /* Source-file line number */
} ReaderObj;
static PyTypeObject Reader_Type;
#define ReaderObject_Check(v) (Py_TYPE(v) == &Reader_Type)
typedef struct {
PyObject_HEAD
PyObject *writeline; /* write output lines to this file */
DialectObj *dialect; /* parsing dialect */
Py_UNICODE *rec; /* buffer for parser.join */
Py_ssize_t rec_size; /* size of allocated record */
Py_ssize_t rec_len; /* length of record */
int num_fields; /* number of fields in record */
} WriterObj;
static PyTypeObject Writer_Type;
/*
* DIALECT class
*/
static PyObject *
get_dialect_from_registry(PyObject * name_obj)
{
PyObject *dialect_obj;
dialect_obj = PyDict_GetItem(dialects, name_obj);
if (dialect_obj == NULL) {
if (!PyErr_Occurred())
PyErr_Format(error_obj, "unknown dialect");
}
else
Py_INCREF(dialect_obj);
return dialect_obj;
}
static PyObject *
get_string(PyObject *str)
{
Py_XINCREF(str);
return str;
}
static PyObject *
get_nullchar_as_None(Py_UNICODE c)
{
if (c == '\0') {
Py_INCREF(Py_None);
return Py_None;
}
else
return PyUnicode_FromUnicode((Py_UNICODE *)&c, 1);
}
static PyObject *
Dialect_get_lineterminator(DialectObj *self)
{
return get_string(self->lineterminator);
}
static PyObject *
Dialect_get_delimiter(DialectObj *self)
{
return get_nullchar_as_None(self->delimiter);
}
static PyObject *
Dialect_get_escapechar(DialectObj *self)
{
return get_nullchar_as_None(self->escapechar);
}
static PyObject *
Dialect_get_quotechar(DialectObj *self)
{
return get_nullchar_as_None(self->quotechar);
}
static PyObject *
Dialect_get_quoting(DialectObj *self)
{
return PyLong_FromLong(self->quoting);
}
static int
_set_bool(const char *name, int *target, PyObject *src, int dflt)
{
if (src == NULL)
*target = dflt;
else {
int b = PyObject_IsTrue(src);
if (b < 0)
return -1;
*target = b;
}
return 0;
}
static int
_set_int(const char *name, int *target, PyObject *src, int dflt)
{
if (src == NULL)
*target = dflt;
else {
long value;
if (!PyLong_CheckExact(src)) {
PyErr_Format(PyExc_TypeError,
"\"%s\" must be an integer", name);
return -1;
}
value = PyLong_AsLong(src);
if (value == -1 && PyErr_Occurred())
return -1;
#if SIZEOF_LONG > SIZEOF_INT
if (value > INT_MAX || value < INT_MIN) {
PyErr_Format(PyExc_ValueError,
"integer out of range for \"%s\"", name);
return -1;
}
#endif
*target = (int)value;
}
return 0;
}
static int
_set_char(const char *name, Py_UNICODE *target, PyObject *src, Py_UNICODE dflt)
{
if (src == NULL)
*target = dflt;
else {
*target = '\0';
if (src != Py_None) {
Py_UNICODE *buf;
Py_ssize_t len;
buf = PyUnicode_AsUnicode(src);
len = PyUnicode_GetSize(src);
if (buf == NULL || len > 1) {
PyErr_Format(PyExc_TypeError,
"\"%s\" must be an 1-character string",
name);
return -1;
}
if (len > 0)
*target = buf[0];
}
}
return 0;
}
static int
_set_str(const char *name, PyObject **target, PyObject *src, const char *dflt)
{
if (src == NULL)
*target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL);
else {
if (src == Py_None)
*target = NULL;
else if (!IS_BASESTRING(src)) {
PyErr_Format(PyExc_TypeError,
"\"%s\" must be a string", name);
return -1;
}
else {
Py_XDECREF(*target);
Py_INCREF(src);
*target = src;
}
}
return 0;
}
static int
dialect_check_quoting(int quoting)
{
StyleDesc *qs;
for (qs = quote_styles; qs->name; qs++) {
if (qs->style == quoting)
return 0;
}
PyErr_Format(PyExc_TypeError, "bad \"quoting\" value");
return -1;
}
#define D_OFF(x) offsetof(DialectObj, x)
static struct PyMemberDef Dialect_memberlist[] = {
{ "skipinitialspace", T_INT, D_OFF(skipinitialspace), READONLY },
{ "doublequote", T_INT, D_OFF(doublequote), READONLY },
{ "strict", T_INT, D_OFF(strict), READONLY },
{ NULL }
};
static PyGetSetDef Dialect_getsetlist[] = {
{ "delimiter", (getter)Dialect_get_delimiter},
{ "escapechar", (getter)Dialect_get_escapechar},
{ "lineterminator", (getter)Dialect_get_lineterminator},
{ "quotechar", (getter)Dialect_get_quotechar},
{ "quoting", (getter)Dialect_get_quoting},
{NULL},
};
static void
Dialect_dealloc(DialectObj *self)
{
Py_XDECREF(self->lineterminator);
Py_TYPE(self)->tp_free((PyObject *)self);
}
static char *dialect_kws[] = {
"dialect",
"delimiter",
"doublequote",
"escapechar",
"lineterminator",
"quotechar",
"quoting",
"skipinitialspace",
"strict",
NULL
};
static PyObject *
dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
DialectObj *self;
PyObject *ret = NULL;
PyObject *dialect = NULL;
PyObject *delimiter = NULL;
PyObject *doublequote = NULL;
PyObject *escapechar = NULL;
PyObject *lineterminator = NULL;
PyObject *quotechar = NULL;
PyObject *quoting = NULL;
PyObject *skipinitialspace = NULL;
PyObject *strict = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"|OOOOOOOOO", dialect_kws,
&dialect,
&delimiter,
&doublequote,
&escapechar,
&lineterminator,
"echar,
"ing,
&skipinitialspace,
&strict))
return NULL;
if (dialect != NULL) {
if (IS_BASESTRING(dialect)) {
dialect = get_dialect_from_registry(dialect);
if (dialect == NULL)
return NULL;
}
else
Py_INCREF(dialect);
/* Can we reuse this instance? */
if (PyObject_TypeCheck(dialect, &Dialect_Type) &&
delimiter == 0 &&
doublequote == 0 &&
escapechar == 0 &&
lineterminator == 0 &&
quotechar == 0 &&
quoting == 0 &&
skipinitialspace == 0 &&
strict == 0)
return dialect;
}
self = (DialectObj *)type->tp_alloc(type, 0);
if (self == NULL) {
Py_XDECREF(dialect);
return NULL;
}
self->lineterminator = NULL;
Py_XINCREF(delimiter);
Py_XINCREF(doublequote);
Py_XINCREF(escapechar);
Py_XINCREF(lineterminator);
Py_XINCREF(quotechar);
Py_XINCREF(quoting);
Py_XINCREF(skipinitialspace);
Py_XINCREF(strict);
if (dialect != NULL) {
#define DIALECT_GETATTR(v, n) \
if (v == NULL) \
v = PyObject_GetAttrString(dialect, n)
DIALECT_GETATTR(delimiter, "delimiter");
DIALECT_GETATTR(doublequote, "doublequote");
DIALECT_GETATTR(escapechar, "escapechar");
DIALECT_GETATTR(lineterminator, "lineterminator");
DIALECT_GETATTR(quotechar, "quotechar");
DIALECT_GETATTR(quoting, "quoting");
DIALECT_GETATTR(skipinitialspace, "skipinitialspace");
DIALECT_GETATTR(strict, "strict");
PyErr_Clear();
}
/* check types and convert to C values */
#define DIASET(meth, name, target, src, dflt) \
if (meth(name, target, src, dflt)) \
goto err
DIASET(_set_char, "delimiter", &self->delimiter, delimiter, ',');
DIASET(_set_bool, "doublequote", &self->doublequote, doublequote, 1);
DIASET(_set_char, "escapechar", &self->escapechar, escapechar, 0);
DIASET(_set_str, "lineterminator", &self->lineterminator, lineterminator, "\r\n");
DIASET(_set_char, "quotechar", &self->quotechar, quotechar, '"');
DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL);
DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, 0);
DIASET(_set_bool, "strict", &self->strict, strict, 0);
/* validate options */
if (dialect_check_quoting(self->quoting))
goto err;
if (self->delimiter == 0) {
PyErr_SetString(PyExc_TypeError, "delimiter must be set");
goto err;
}
if (quotechar == Py_None && quoting == NULL)
self->quoting = QUOTE_NONE;
if (self->quoting != QUOTE_NONE && self->quotechar == 0) {
PyErr_SetString(PyExc_TypeError,
"quotechar must be set if quoting enabled");
goto err;
}
if (self->lineterminator == 0) {
PyErr_SetString(PyExc_TypeError, "lineterminator must be set");
goto err;
}
ret = (PyObject *)self;
Py_INCREF(self);
err:
Py_XDECREF(self);
Py_XDECREF(dialect);
Py_XDECREF(delimiter);
Py_XDECREF(doublequote);
Py_XDECREF(escapechar);
Py_XDECREF(lineterminator);
Py_XDECREF(quotechar);
Py_XDECREF(quoting);
Py_XDECREF(skipinitialspace);
Py_XDECREF(strict);
return ret;
}
PyDoc_STRVAR(Dialect_Type_doc,
"CSV dialect\n"
"\n"
"The Dialect type records CSV parsing and generation options.\n");
static PyTypeObject Dialect_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_csv.Dialect", /* tp_name */
sizeof(DialectObj), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)Dialect_dealloc, /* tp_dealloc */
(printfunc)0, /* tp_print */
(getattrfunc)0, /* tp_getattr */
(setattrfunc)0, /* tp_setattr */
0, /* tp_reserved */
(reprfunc)0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
(hashfunc)0, /* tp_hash */
(ternaryfunc)0, /* tp_call */
(reprfunc)0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
Dialect_Type_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
Dialect_memberlist, /* tp_members */
Dialect_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
dialect_new, /* tp_new */
0, /* tp_free */
};
/*
* Return an instance of the dialect type, given a Python instance or kwarg
* description of the dialect
*/
static PyObject *
_call_dialect(PyObject *dialect_inst, PyObject *kwargs)
{
PyObject *ctor_args;
PyObject *dialect;
ctor_args = Py_BuildValue(dialect_inst ? "(O)" : "()", dialect_inst);
if (ctor_args == NULL)
return NULL;
dialect = PyObject_Call((PyObject *)&Dialect_Type, ctor_args, kwargs);
Py_DECREF(ctor_args);
return dialect;
}
/*
* READER
*/
static int
parse_save_field(ReaderObj *self)
{
PyObject *field;
field = PyUnicode_FromUnicode(self->field, self->field_len);
if (field == NULL)
return -1;
self->field_len = 0;
if (self->numeric_field) {
PyObject *tmp;
self->numeric_field = 0;
tmp = PyNumber_Float(field);
if (tmp == NULL) {
Py_DECREF(field);
return -1;
}
Py_DECREF(field);
field = tmp;
}
PyList_Append(self->fields, field);
Py_DECREF(field);
return 0;
}
static int
parse_grow_buff(ReaderObj *self)
{
if (self->field_size == 0) {
self->field_size = 4096;
if (self->field != NULL)
PyMem_Free(self->field);
self->field = PyMem_New(Py_UNICODE, self->field_size);
}
else {
if (self->field_size > PY_SSIZE_T_MAX / 2) {
PyErr_NoMemory();
return 0;
}
self->field_size *= 2;
self->field = PyMem_Resize(self->field, Py_UNICODE,
self->field_size);
}
if (self->field == NULL) {
PyErr_NoMemory();
return 0;
}
return 1;
}
static int
parse_add_char(ReaderObj *self, Py_UNICODE c)
{
if (self->field_len >= field_limit) {
PyErr_Format(error_obj, "field larger than field limit (%ld)",
field_limit);
return -1;
}
if (self->field_len == self->field_size && !parse_grow_buff(self))
return -1;
self->field[self->field_len++] = c;
return 0;
}
static int
parse_process_char(ReaderObj *self, Py_UNICODE c)
{
DialectObj *dialect = self->dialect;
switch (self->state) {
case START_RECORD:
/* start of record */
if (c == '\0')
/* empty line - return [] */
break;
else if (c == '\n' || c == '\r') {
self->state = EAT_CRNL;
break;
}
/* normal character - handle as START_FIELD */
self->state = START_FIELD;
/* fallthru */
case START_FIELD:
/* expecting field */
if (c == '\n' || c == '\r' || c == '\0') {
/* save empty field - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (c == dialect->quotechar &&
dialect->quoting != QUOTE_NONE) {
/* start quoted field */
self->state = IN_QUOTED_FIELD;
}
else if (c == dialect->escapechar) {
/* possible escaped character */
self->state = ESCAPED_CHAR;
}
else if (c == ' ' && dialect->skipinitialspace)
/* ignore space at start of field */
;
else if (c == dialect->delimiter) {
/* save empty field */
if (parse_save_field(self) < 0)
return -1;
}
else {
/* begin new unquoted field */
if (dialect->quoting == QUOTE_NONNUMERIC)
self->numeric_field = 1;
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_FIELD;
}
break;
case ESCAPED_CHAR:
if (c == '\0')
c = '\n';
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_FIELD;
break;
case IN_FIELD:
/* in unquoted field */
if (c == '\n' || c == '\r' || c == '\0') {
/* end of line - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (c == dialect->escapechar) {
/* possible escaped character */
self->state = ESCAPED_CHAR;
}
else if (c == dialect->delimiter) {
/* save field - wait for new field */
if (parse_save_field(self) < 0)
return -1;
self->state = START_FIELD;
}
else {
/* normal character - save in field */
if (parse_add_char(self, c) < 0)
return -1;
}
break;
case IN_QUOTED_FIELD:
/* in quoted field */
if (c == '\0')
;
else if (c == dialect->escapechar) {
/* Possible escape character */
self->state = ESCAPE_IN_QUOTED_FIELD;
}
else if (c == dialect->quotechar &&
dialect->quoting != QUOTE_NONE) {
if (dialect->doublequote) {
/* doublequote; " represented by "" */
self->state = QUOTE_IN_QUOTED_FIELD;
}
else {
/* end of quote part of field */
self->state = IN_FIELD;
}
}
else {
/* normal character - save in field */
if (parse_add_char(self, c) < 0)
return -1;
}
break;
case ESCAPE_IN_QUOTED_FIELD:
if (c == '\0')
c = '\n';
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_QUOTED_FIELD;
break;
case QUOTE_IN_QUOTED_FIELD:
/* doublequote - seen a quote in an quoted field */
if (dialect->quoting != QUOTE_NONE &&
c == dialect->quotechar) {
/* save "" as " */
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_QUOTED_FIELD;
}
else if (c == dialect->delimiter) {
/* save field - wait for new field */
if (parse_save_field(self) < 0)
return -1;
self->state = START_FIELD;
}
else if (c == '\n' || c == '\r' || c == '\0') {
/* end of line - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (!dialect->strict) {
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_FIELD;
}
else {
/* illegal */
PyErr_Format(error_obj, "'%c' expected after '%c'",
dialect->delimiter,
dialect->quotechar);
return -1;
}
break;
case EAT_CRNL:
if (c == '\n' || c == '\r')
;
else if (c == '\0')
self->state = START_RECORD;
else {
PyErr_Format(error_obj, "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?");
return -1;
}
break;
}
return 0;
}
static int
parse_reset(ReaderObj *self)
{
Py_XDECREF(self->fields);
self->fields = PyList_New(0);
if (self->fields == NULL)
return -1;
self->field_len = 0;
self->state = START_RECORD;
self->numeric_field = 0;
return 0;
}
static PyObject *
Reader_iternext(ReaderObj *self)
{
PyObject *lineobj;
PyObject *fields = NULL;
Py_UNICODE *line, c;
Py_ssize_t linelen;
if (parse_reset(self) < 0)
return NULL;
do {
lineobj = PyIter_Next(self->input_iter);
if (lineobj == NULL) {
/* End of input OR exception */
if (!PyErr_Occurred() && self->field_len != 0)
PyErr_Format(error_obj,
"newline inside string");
return NULL;
}
if (!PyUnicode_Check(lineobj)) {
PyErr_Format(error_obj,
"iterator should return strings, "
"not %.200s "
"(did you open the file in text mode?)",
lineobj->ob_type->tp_name
);
Py_DECREF(lineobj);
return NULL;
}
++self->line_num;
line = PyUnicode_AsUnicode(lineobj);
linelen = PyUnicode_GetSize(lineobj);
if (line == NULL || linelen < 0) {
Py_DECREF(lineobj);
return NULL;
}
while (linelen--) {
c = *line++;
if (c == '\0') {
Py_DECREF(lineobj);
PyErr_Format(error_obj,
"line contains NULL byte");
goto err;
}
if (parse_process_char(self, c) < 0) {
Py_DECREF(lineobj);
goto err;
}
}
Py_DECREF(lineobj);
if (parse_process_char(self, 0) < 0)
goto err;
} while (self->state != START_RECORD);
fields = self->fields;
self->fields = NULL;
err:
return fields;
}
static void
Reader_dealloc(ReaderObj *self)
{
PyObject_GC_UnTrack(self);
Py_XDECREF(self->dialect);
Py_XDECREF(self->input_iter);
Py_XDECREF(self->fields);
if (self->field != NULL)
PyMem_Free(self->field);
PyObject_GC_Del(self);
}
static int
Reader_traverse(ReaderObj *self, visitproc visit, void *arg)
{
Py_VISIT(self->dialect);
Py_VISIT(self->input_iter);
Py_VISIT(self->fields);
return 0;
}
static int
Reader_clear(ReaderObj *self)
{
Py_CLEAR(self->dialect);
Py_CLEAR(self->input_iter);
Py_CLEAR(self->fields);
return 0;
}
PyDoc_STRVAR(Reader_Type_doc,
"CSV reader\n"
"\n"
"Reader objects are responsible for reading and parsing tabular data\n"
"in CSV format.\n"
);
static struct PyMethodDef Reader_methods[] = {
{ NULL, NULL }
};
#define R_OFF(x) offsetof(ReaderObj, x)
static struct PyMemberDef Reader_memberlist[] = {
{ "dialect", T_OBJECT, R_OFF(dialect), READONLY },
{ "line_num", T_ULONG, R_OFF(line_num), READONLY },
{ NULL }
};
static PyTypeObject Reader_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_csv.reader", /*tp_name*/
sizeof(ReaderObj), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor)Reader_dealloc, /*tp_dealloc*/
(printfunc)0, /*tp_print*/
(getattrfunc)0, /*tp_getattr*/
(setattrfunc)0, /*tp_setattr*/
0, /*tp_reserved*/
(reprfunc)0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC, /*tp_flags*/
Reader_Type_doc, /*tp_doc*/
(traverseproc)Reader_traverse, /*tp_traverse*/
(inquiry)Reader_clear, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
PyObject_SelfIter, /*tp_iter*/
(getiterfunc)Reader_iternext, /*tp_iternext*/
Reader_methods, /*tp_methods*/
Reader_memberlist, /*tp_members*/
0, /*tp_getset*/
};
static PyObject *
csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args)
{
PyObject * iterator, * dialect = NULL;
ReaderObj * self = PyObject_GC_New(ReaderObj, &Reader_Type);
if (!self)
return NULL;
self->dialect = NULL;
self->fields = NULL;
self->input_iter = NULL;
self->field = NULL;
self->field_size = 0;
self->line_num = 0;
if (parse_reset(self) < 0) {
Py_DECREF(self);
return NULL;
}
if (!PyArg_UnpackTuple(args, "", 1, 2, &iterator, &dialect)) {
Py_DECREF(self);
return NULL;
}
self->input_iter = PyObject_GetIter(iterator);
if (self->input_iter == NULL) {
PyErr_SetString(PyExc_TypeError,
"argument 1 must be an iterator");
Py_DECREF(self);
return NULL;
}
self->dialect = (DialectObj *)_call_dialect(dialect, keyword_args);
if (self->dialect == NULL) {
Py_DECREF(self);
return NULL;
}
PyObject_GC_Track(self);
return (PyObject *)self;
}
/*
* WRITER
*/
/* ---------------------------------------------------------------- */
static void
join_reset(WriterObj *self)
{
self->rec_len = 0;
self->num_fields = 0;
}
#define MEM_INCR 32768
/* Calculate new record length or append field to record. Return new
* record length.
*/
static Py_ssize_t
join_append_data(WriterObj *self, Py_UNICODE *field, int quote_empty,
int *quoted, int copy_phase)
{
DialectObj *dialect = self->dialect;
int i;
Py_ssize_t rec_len;
Py_UNICODE *lineterm;
#define ADDCH(c) \
do {\
if (copy_phase) \
self->rec[rec_len] = c;\
rec_len++;\
} while(0)
lineterm = PyUnicode_AsUnicode(dialect->lineterminator);
if (lineterm == NULL)
return -1;
rec_len = self->rec_len;
/* If this is not the first field we need a field separator */
if (self->num_fields > 0)
ADDCH(dialect->delimiter);
/* Handle preceding quote */
if (copy_phase && *quoted)
ADDCH(dialect->quotechar);
/* Copy/count field data */
/* If field is null just pass over */
for (i = 0; field; i++) {
Py_UNICODE c = field[i];
int want_escape = 0;
if (c == '\0')
break;
if (c == dialect->delimiter ||
c == dialect->escapechar ||
c == dialect->quotechar ||
Py_UNICODE_strchr(lineterm, c)) {
if (dialect->quoting == QUOTE_NONE)
want_escape = 1;
else {
if (c == dialect->quotechar) {
if (dialect->doublequote)
ADDCH(dialect->quotechar);
else
want_escape = 1;
}
if (!want_escape)
*quoted = 1;
}
if (want_escape) {
if (!dialect->escapechar) {
PyErr_Format(error_obj,
"need to escape, but no escapechar set");
return -1;
}
ADDCH(dialect->escapechar);
}
}
/* Copy field character into record buffer.
*/
ADDCH(c);
}
/* If field is empty check if it needs to be quoted.
*/
if (i == 0 && quote_empty) {
if (dialect->quoting == QUOTE_NONE) {
PyErr_Format(error_obj,
"single empty field record must be quoted");
return -1;
}
else
*quoted = 1;
}
if (*quoted) {
if (copy_phase)
ADDCH(dialect->quotechar);
else
rec_len += 2;
}
return rec_len;
#undef ADDCH
}
static int
join_check_rec_size(WriterObj *self, Py_ssize_t rec_len)
{
if (rec_len < 0 || rec_len > PY_SSIZE_T_MAX - MEM_INCR) {
PyErr_NoMemory();
return 0;
}
if (rec_len > self->rec_size) {
if (self->rec_size == 0) {
self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;
if (self->rec != NULL)
PyMem_Free(self->rec);
self->rec = PyMem_New(Py_UNICODE, self->rec_size);
}
else {
Py_UNICODE* old_rec = self->rec;
self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;
self->rec = PyMem_Resize(self->rec, Py_UNICODE,
self->rec_size);
if (self->rec == NULL)
PyMem_Free(old_rec);
}
if (self->rec == NULL) {
PyErr_NoMemory();
return 0;
}
}
return 1;
}
static int
join_append(WriterObj *self, Py_UNICODE *field, int *quoted, int quote_empty)
{
Py_ssize_t rec_len;
rec_len = join_append_data(self, field, quote_empty, quoted, 0);
if (rec_len < 0)
return 0;
/* grow record buffer if necessary */
if (!join_check_rec_size(self, rec_len))
return 0;
self->rec_len = join_append_data(self, field, quote_empty, quoted, 1);
self->num_fields++;
return 1;
}
static int
join_append_lineterminator(WriterObj *self)
{
Py_ssize_t terminator_len;
Py_UNICODE *terminator;
terminator_len = PyUnicode_GetSize(self->dialect->lineterminator);
if (terminator_len == -1)
return 0;
/* grow record buffer if necessary */
if (!join_check_rec_size(self, self->rec_len + terminator_len))
return 0;
terminator = PyUnicode_AsUnicode(self->dialect->lineterminator);
if (terminator == NULL)
return 0;
memmove(self->rec + self->rec_len, terminator,
sizeof(Py_UNICODE)*terminator_len);
self->rec_len += terminator_len;
return 1;
}
PyDoc_STRVAR(csv_writerow_doc,
"writerow(sequence)\n"
"\n"
"Construct and write a CSV record from a sequence of fields. Non-string\n"
"elements will be converted to string.");
static PyObject *
csv_writerow(WriterObj *self, PyObject *seq)
{
DialectObj *dialect = self->dialect;
Py_ssize_t len, i;
if (!PySequence_Check(seq))
return PyErr_Format(error_obj, "sequence expected");
len = PySequence_Length(seq);
if (len < 0)
return NULL;
/* Join all fields in internal buffer.
*/
join_reset(self);
for (i = 0; i < len; i++) {
PyObject *field;
int append_ok;
int quoted;
field = PySequence_GetItem(seq, i);
if (field == NULL)
return NULL;
switch (dialect->quoting) {
case QUOTE_NONNUMERIC:
quoted = !PyNumber_Check(field);
break;
case QUOTE_ALL:
quoted = 1;
break;
default:
quoted = 0;
break;
}
if (PyUnicode_Check(field)) {
append_ok = join_append(self,
PyUnicode_AS_UNICODE(field),
"ed, len == 1);
Py_DECREF(field);
}
else if (field == Py_None) {
append_ok = join_append(self, NULL,
"ed, len == 1);
Py_DECREF(field);
}
else {
PyObject *str;
str = PyObject_Str(field);
Py_DECREF(field);
if (str == NULL)
return NULL;
append_ok = join_append(self,
PyUnicode_AS_UNICODE(str),
"ed, len == 1);
Py_DECREF(str);
}
if (!append_ok)
return NULL;
}
/* Add line terminator.
*/
if (!join_append_lineterminator(self))
return 0;
return PyObject_CallFunction(self->writeline,
"(u#)", self->rec,
self->rec_len);
}
PyDoc_STRVAR(csv_writerows_doc,
"writerows(sequence of sequences)\n"
"\n"
"Construct and write a series of sequences to a csv file. Non-string\n"
"elements will be converted to string.");
static PyObject *
csv_writerows(WriterObj *self, PyObject *seqseq)
{
PyObject *row_iter, *row_obj, *result;
row_iter = PyObject_GetIter(seqseq);
if (row_iter == NULL) {
PyErr_SetString(PyExc_TypeError,
"writerows() argument must be iterable");
return NULL;
}
while ((row_obj = PyIter_Next(row_iter))) {
result = csv_writerow(self, row_obj);
Py_DECREF(row_obj);
if (!result) {
Py_DECREF(row_iter);
return NULL;
}
else
Py_DECREF(result);
}
Py_DECREF(row_iter);
if (PyErr_Occurred())
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef Writer_methods[] = {
{ "writerow", (PyCFunction)csv_writerow, METH_O, csv_writerow_doc},
{ "writerows", (PyCFunction)csv_writerows, METH_O, csv_writerows_doc},
{ NULL, NULL }
};
#define W_OFF(x) offsetof(WriterObj, x)
static struct PyMemberDef Writer_memberlist[] = {
{ "dialect", T_OBJECT, W_OFF(dialect), READONLY },
{ NULL }
};
static void
Writer_dealloc(WriterObj *self)
{
PyObject_GC_UnTrack(self);
Py_XDECREF(self->dialect);
Py_XDECREF(self->writeline);
if (self->rec != NULL)
PyMem_Free(self->rec);
PyObject_GC_Del(self);
}
static int
Writer_traverse(WriterObj *self, visitproc visit, void *arg)
{
Py_VISIT(self->dialect);
Py_VISIT(self->writeline);
return 0;
}
static int
Writer_clear(WriterObj *self)
{
Py_CLEAR(self->dialect);
Py_CLEAR(self->writeline);
return 0;
}
PyDoc_STRVAR(Writer_Type_doc,
"CSV writer\n"
"\n"
"Writer objects are responsible for generating tabular data\n"
"in CSV format from sequence input.\n"
);
static PyTypeObject Writer_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_csv.writer", /*tp_name*/
sizeof(WriterObj), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor)Writer_dealloc, /*tp_dealloc*/
(printfunc)0, /*tp_print*/
(getattrfunc)0, /*tp_getattr*/
(setattrfunc)0, /*tp_setattr*/
0, /*tp_reserved*/
(reprfunc)0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC, /*tp_flags*/
Writer_Type_doc,
(traverseproc)Writer_traverse, /*tp_traverse*/
(inquiry)Writer_clear, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
(getiterfunc)0, /*tp_iter*/
(getiterfunc)0, /*tp_iternext*/
Writer_methods, /*tp_methods*/
Writer_memberlist, /*tp_members*/
0, /*tp_getset*/
};
static PyObject *
csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args)
{
PyObject * output_file, * dialect = NULL;
WriterObj * self = PyObject_GC_New(WriterObj, &Writer_Type);
if (!self)
return NULL;
self->dialect = NULL;
self->writeline = NULL;
self->rec = NULL;
self->rec_size = 0;
self->rec_len = 0;
self->num_fields = 0;
if (!PyArg_UnpackTuple(args, "", 1, 2, &output_file, &dialect)) {
Py_DECREF(self);
return NULL;
}
self->writeline = PyObject_GetAttrString(output_file, "write");
if (self->writeline == NULL || !PyCallable_Check(self->writeline)) {
PyErr_SetString(PyExc_TypeError,
"argument 1 must have a \"write\" method");
Py_DECREF(self);
return NULL;
}
self->dialect = (DialectObj *)_call_dialect(dialect, keyword_args);
if (self->dialect == NULL) {
Py_DECREF(self);
return NULL;
}
PyObject_GC_Track(self);
return (PyObject *)self;
}
/*
* DIALECT REGISTRY
*/
static PyObject *
csv_list_dialects(PyObject *module, PyObject *args)
{
return PyDict_Keys(dialects);
}
static PyObject *
csv_register_dialect(PyObject *module, PyObject *args, PyObject *kwargs)
{
PyObject *name_obj, *dialect_obj = NULL;
PyObject *dialect;
if (!PyArg_UnpackTuple(args, "", 1, 2, &name_obj, &dialect_obj))
return NULL;
if (!IS_BASESTRING(name_obj)) {
PyErr_SetString(PyExc_TypeError,
"dialect name must be a string or unicode");
return NULL;
}
dialect = _call_dialect(dialect_obj, kwargs);
if (dialect == NULL)
return NULL;
if (PyDict_SetItem(dialects, name_obj, dialect) < 0) {
Py_DECREF(dialect);
return NULL;
}
Py_DECREF(dialect);
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
csv_unregister_dialect(PyObject *module, PyObject *name_obj)
{
if (PyDict_DelItem(dialects, name_obj) < 0)
return PyErr_Format(error_obj, "unknown dialect");
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
csv_get_dialect(PyObject *module, PyObject *name_obj)
{
return get_dialect_from_registry(name_obj);
}
static PyObject *
csv_field_size_limit(PyObject *module, PyObject *args)
{
PyObject *new_limit = NULL;
long old_limit = field_limit;
if (!PyArg_UnpackTuple(args, "field_size_limit", 0, 1, &new_limit))
return NULL;
if (new_limit != NULL) {
if (!PyLong_CheckExact(new_limit)) {
PyErr_Format(PyExc_TypeError,
"limit must be an integer");
return NULL;
}
field_limit = PyLong_AsLong(new_limit);
if (field_limit == -1 && PyErr_Occurred()) {
field_limit = old_limit;
return NULL;
}
}
return PyLong_FromLong(old_limit);
}
/*
* MODULE
*/
PyDoc_STRVAR(csv_module_doc,
"CSV parsing and writing.\n"
"\n"
"This module provides classes that assist in the reading and writing\n"
"of Comma Separated Value (CSV) files, and implements the interface\n"
"described by PEP 305. Although many CSV files are simple to parse,\n"
"the format is not formally defined by a stable specification and\n"
"is subtle enough that parsing lines of a CSV file with something\n"
"like line.split(\",\") is bound to fail. The module supports three\n"
"basic APIs: reading, writing, and registration of dialects.\n"
"\n"
"\n"
"DIALECT REGISTRATION:\n"
"\n"
"Readers and writers support a dialect argument, which is a convenient\n"
"handle on a group of settings. When the dialect argument is a string,\n"
"it identifies one of the dialects previously registered with the module.\n"
"If it is a class or instance, the attributes of the argument are used as\n"
"the settings for the reader or writer:\n"
"\n"
" class excel:\n"
" delimiter = ','\n"
" quotechar = '\"'\n"
" escapechar = None\n"
" doublequote = True\n"
" skipinitialspace = False\n"
" lineterminator = '\\r\\n'\n"
" quoting = QUOTE_MINIMAL\n"
"\n"
"SETTINGS:\n"
"\n"
" * quotechar - specifies a one-character string to use as the \n"
" quoting character. It defaults to '\"'.\n"
" * delimiter - specifies a one-character string to use as the \n"
" field separator. It defaults to ','.\n"
" * skipinitialspace - specifies how to interpret whitespace which\n"
" immediately follows a delimiter. It defaults to False, which\n"
" means that whitespace immediately following a delimiter is part\n"
" of the following field.\n"
" * lineterminator - specifies the character sequence which should \n"
" terminate rows.\n"
" * quoting - controls when quotes should be generated by the writer.\n"
" It can take on any of the following module constants:\n"
"\n"
" csv.QUOTE_MINIMAL means only when required, for example, when a\n"
" field contains either the quotechar or the delimiter\n"
" csv.QUOTE_ALL means that quotes are always placed around fields.\n"
" csv.QUOTE_NONNUMERIC means that quotes are always placed around\n"
" fields which do not parse as integers or floating point\n"
" numbers.\n"
" csv.QUOTE_NONE means that quotes are never placed around fields.\n"
" * escapechar - specifies a one-character string used to escape \n"
" the delimiter when quoting is set to QUOTE_NONE.\n"
" * doublequote - controls the handling of quotes inside fields. When\n"
" True, two consecutive quotes are interpreted as one during read,\n"
" and when writing, each quote character embedded in the data is\n"
" written as two quotes\n");
PyDoc_STRVAR(csv_reader_doc,
" csv_reader = reader(iterable [, dialect='excel']\n"
" [optional keyword args])\n"
" for row in csv_reader:\n"
" process(row)\n"
"\n"
"The \"iterable\" argument can be any object that returns a line\n"
"of input for each iteration, such as a file object or a list. The\n"
"optional \"dialect\" parameter is discussed below. The function\n"
"also accepts optional keyword arguments which override settings\n"
"provided by the dialect.\n"
"\n"
"The returned object is an iterator. Each iteration returns a row\n"
"of the CSV file (which can span multiple input lines):\n");
PyDoc_STRVAR(csv_writer_doc,
" csv_writer = csv.writer(fileobj [, dialect='excel']\n"
" [optional keyword args])\n"
" for row in sequence:\n"
" csv_writer.writerow(row)\n"
"\n"
" [or]\n"
"\n"
" csv_writer = csv.writer(fileobj [, dialect='excel']\n"
" [optional keyword args])\n"
" csv_writer.writerows(rows)\n"
"\n"
"The \"fileobj\" argument can be any object that supports the file API.\n");
PyDoc_STRVAR(csv_list_dialects_doc,
"Return a list of all know dialect names.\n"
" names = csv.list_dialects()");
PyDoc_STRVAR(csv_get_dialect_doc,
"Return the dialect instance associated with name.\n"
" dialect = csv.get_dialect(name)");
PyDoc_STRVAR(csv_register_dialect_doc,
"Create a mapping from a string name to a dialect class.\n"
" dialect = csv.register_dialect(name, dialect)");
PyDoc_STRVAR(csv_unregister_dialect_doc,
"Delete the name/dialect mapping associated with a string name.\n"
" csv.unregister_dialect(name)");
PyDoc_STRVAR(csv_field_size_limit_doc,
"Sets an upper limit on parsed fields.\n"
" csv.field_size_limit([limit])\n"
"\n"
"Returns old limit. If limit is not given, no new limit is set and\n"
"the old limit is returned");
static struct PyMethodDef csv_methods[] = {
{ "reader", (PyCFunction)csv_reader,
METH_VARARGS | METH_KEYWORDS, csv_reader_doc},
{ "writer", (PyCFunction)csv_writer,
METH_VARARGS | METH_KEYWORDS, csv_writer_doc},
{ "list_dialects", (PyCFunction)csv_list_dialects,
METH_NOARGS, csv_list_dialects_doc},
{ "register_dialect", (PyCFunction)csv_register_dialect,
METH_VARARGS | METH_KEYWORDS, csv_register_dialect_doc},
{ "unregister_dialect", (PyCFunction)csv_unregister_dialect,
METH_O, csv_unregister_dialect_doc},
{ "get_dialect", (PyCFunction)csv_get_dialect,
METH_O, csv_get_dialect_doc},
{ "field_size_limit", (PyCFunction)csv_field_size_limit,
METH_VARARGS, csv_field_size_limit_doc},
{ NULL, NULL }
};
static struct PyModuleDef _csvmodule = {
PyModuleDef_HEAD_INIT,
"_csv",
csv_module_doc,
-1,
csv_methods,
NULL,
NULL,
NULL,
NULL
};
PyMODINIT_FUNC
PyInit__csv(void)
{
PyObject *module;
StyleDesc *style;
if (PyType_Ready(&Dialect_Type) < 0)
return NULL;
if (PyType_Ready(&Reader_Type) < 0)
return NULL;
if (PyType_Ready(&Writer_Type) < 0)
return NULL;
/* Create the module and add the functions */
module = PyModule_Create(&_csvmodule);
if (module == NULL)
return NULL;
/* Add version to the module. */
if (PyModule_AddStringConstant(module, "__version__",
MODULE_VERSION) == -1)
return NULL;
/* Add _dialects dictionary */
dialects = PyDict_New();
if (dialects == NULL)
return NULL;
if (PyModule_AddObject(module, "_dialects", dialects))
return NULL;
/* Add quote styles into dictionary */
for (style = quote_styles; style->name; style++) {
if (PyModule_AddIntConstant(module, style->name,
style->style) == -1)
return NULL;
}
/* Add the Dialect type */
Py_INCREF(&Dialect_Type);
if (PyModule_AddObject(module, "Dialect", (PyObject *)&Dialect_Type))
return NULL;
/* Add the CSV exception object to the module. */
error_obj = PyErr_NewException("_csv.Error", NULL, NULL);
if (error_obj == NULL)
return NULL;
PyModule_AddObject(module, "Error", error_obj);
return module;
}
|
