diff options
Diffstat (limited to 'Objects')
| -rw-r--r-- | Objects/dictobject.c | 190 | ||||
| -rw-r--r-- | Objects/exceptions.c | 49 | ||||
| -rw-r--r-- | Objects/longobject.c | 88 | ||||
| -rw-r--r-- | Objects/rangeobject.c | 41 | ||||
| -rw-r--r-- | Objects/unicodeobject.c | 91 |
5 files changed, 181 insertions, 278 deletions
diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d8ab91fb12..cb67dda33f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -10,8 +10,9 @@ This implements the dictionary's hashtable. -As of Python 3.6, this is compact and ordered. Basic idea is described here. -https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html +As of Python 3.6, this is compact and ordered. Basic idea is described here: +* https://mail.python.org/pipermail/python-dev/2012-December/123028.html +* https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html layout: @@ -1197,41 +1198,21 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } /* -Internal routine used by dictresize() to insert an item which is -known to be absent from the dict. This routine also assumes that -the dict contains no deleted entries. Besides the performance benefit, -using insertdict() in dictresize() is dangerous (SF bug #1456209). -Note that no refcounts are changed by this routine; if needed, the caller -is responsible for incref'ing `key` and `value`. -Neither mp->ma_used nor k->dk_usable are modified by this routine; the caller -must set them correctly +Internal routine used by dictresize() to buid a hashtable of entries. */ static void -insertdict_clean(PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyObject *value) +build_indices(PyDictKeysObject *keys, PyDictKeyEntry *ep, Py_ssize_t n) { - size_t i; - PyDictKeysObject *k = mp->ma_keys; - size_t mask = (size_t)DK_SIZE(k)-1; - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - PyDictKeyEntry *ep; - - assert(k->dk_lookup != NULL); - assert(value != NULL); - assert(key != NULL); - assert(PyUnicode_CheckExact(key) || k->dk_lookup == lookdict); - i = hash & mask; - for (size_t perturb = hash; dk_get_index(k, i) != DKIX_EMPTY;) { - perturb >>= PERTURB_SHIFT; - i = mask & ((i << 2) + i + perturb + 1); + size_t mask = (size_t)DK_SIZE(keys) - 1; + for (Py_ssize_t ix = 0; ix != n; ix++, ep++) { + Py_hash_t hash = ep->me_hash; + size_t i = hash & mask; + for (size_t perturb = hash; dk_get_index(keys, i) != DKIX_EMPTY;) { + perturb >>= PERTURB_SHIFT; + i = mask & ((i << 2) + i + perturb + 1); + } + dk_set_index(keys, i, ix); } - ep = &ep0[k->dk_nentries]; - assert(ep->me_value == NULL); - dk_set_index(k, i, k->dk_nentries); - k->dk_nentries++; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; } /* @@ -1247,10 +1228,10 @@ but can be resplit by make_keys_shared(). static int dictresize(PyDictObject *mp, Py_ssize_t minused) { - Py_ssize_t i, newsize; + Py_ssize_t newsize, numentries; PyDictKeysObject *oldkeys; PyObject **oldvalues; - PyDictKeyEntry *ep0; + PyDictKeyEntry *oldentries, *newentries; /* Find the smallest table size > minused. */ for (newsize = PyDict_MINSIZE; @@ -1261,8 +1242,14 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) PyErr_NoMemory(); return -1; } + oldkeys = mp->ma_keys; - oldvalues = mp->ma_values; + + /* NOTE: Current odict checks mp->ma_keys to detect resize happen. + * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. + * TODO: Try reusing oldkeys when reimplement odict. + */ + /* Allocate a new table. */ mp->ma_keys = new_keys_object(newsize); if (mp->ma_keys == NULL) { @@ -1271,42 +1258,59 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) } if (oldkeys->dk_lookup == lookdict) mp->ma_keys->dk_lookup = lookdict; - mp->ma_values = NULL; - ep0 = DK_ENTRIES(oldkeys); - /* Main loop below assumes we can transfer refcount to new keys - * and that value is stored in me_value. - * Increment ref-counts and copy values here to compensate - * This (resizing a split table) should be relatively rare */ + + numentries = mp->ma_used; + oldentries = DK_ENTRIES(oldkeys); + newentries = DK_ENTRIES(mp->ma_keys); + oldvalues = mp->ma_values; if (oldvalues != NULL) { - for (i = 0; i < oldkeys->dk_nentries; i++) { - if (oldvalues[i] != NULL) { - Py_INCREF(ep0[i].me_key); - ep0[i].me_value = oldvalues[i]; - } - } - } - /* Main loop */ - for (i = 0; i < oldkeys->dk_nentries; i++) { - PyDictKeyEntry *ep = &ep0[i]; - if (ep->me_value != NULL) { - insertdict_clean(mp, ep->me_key, ep->me_hash, ep->me_value); + /* Convert split table into new combined table. + * We must incref keys; we can transfer values. + * Note that values of split table is always dense. + */ + for (Py_ssize_t i = 0; i < numentries; i++) { + assert(oldvalues[i] != NULL); + PyDictKeyEntry *ep = &oldentries[i]; + PyObject *key = ep->me_key; + Py_INCREF(key); + newentries[i].me_key = key; + newentries[i].me_hash = ep->me_hash; + newentries[i].me_value = oldvalues[i]; } - } - mp->ma_keys->dk_usable -= mp->ma_used; - if (oldvalues != NULL) { - /* NULL out me_value slot in oldkeys, in case it was shared */ - for (i = 0; i < oldkeys->dk_nentries; i++) - ep0[i].me_value = NULL; + DK_DECREF(oldkeys); + mp->ma_values = NULL; if (oldvalues != empty_values) { free_values(oldvalues); } } - else { + else { // combined table. + if (oldkeys->dk_nentries == numentries) { + memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); + } + else { + PyDictKeyEntry *ep = oldentries; + for (Py_ssize_t i = 0; i < numentries; i++) { + while (ep->me_value == NULL) + ep++; + newentries[i] = *ep++; + } + } + assert(oldkeys->dk_lookup != lookdict_split); assert(oldkeys->dk_refcnt == 1); - DK_DEBUG_DECREF PyObject_FREE(oldkeys); + if (oldkeys->dk_size == PyDict_MINSIZE && + numfreekeys < PyDict_MAXFREELIST) { + DK_DEBUG_DECREF keys_free_list[numfreekeys++] = oldkeys; + } + else { + DK_DEBUG_DECREF PyObject_FREE(oldkeys); + } } + + build_indices(mp->ma_keys, newentries, numentries); + mp->ma_keys->dk_usable -= numentries; + mp->ma_keys->dk_nentries = numentries; return 0; } @@ -1687,7 +1691,7 @@ int _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue, Py_hash_t *phash) { - Py_ssize_t i, n; + Py_ssize_t i; PyDictObject *mp; PyDictKeyEntry *entry_ptr; PyObject *value; @@ -1696,21 +1700,18 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; mp = (PyDictObject *)op; i = *ppos; - n = mp->ma_keys->dk_nentries; - if ((size_t)i >= (size_t)n) - return 0; if (mp->ma_values) { - PyObject **value_ptr = &mp->ma_values[i]; - while (i < n && *value_ptr == NULL) { - value_ptr++; - i++; - } - if (i >= n) + if (i < 0 || i >= mp->ma_used) return 0; + /* values of split table is always dense */ entry_ptr = &DK_ENTRIES(mp->ma_keys)[i]; - value = *value_ptr; + value = mp->ma_values[i]; + assert(value != NULL); } else { + Py_ssize_t n = mp->ma_keys->dk_nentries; + if (i < 0 || i >= n) + return 0; entry_ptr = &DK_ENTRIES(mp->ma_keys)[i]; while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; @@ -3375,7 +3376,7 @@ static PyObject* dictiter_iternextkey(dictiterobject *di) { PyObject *key; - Py_ssize_t i, n; + Py_ssize_t i; PyDictKeysObject *k; PyDictObject *d = di->di_dict; @@ -3392,18 +3393,15 @@ dictiter_iternextkey(dictiterobject *di) i = di->di_pos; k = d->ma_keys; - n = k->dk_nentries; + assert(i >= 0); if (d->ma_values) { - PyObject **value_ptr = &d->ma_values[i]; - while (i < n && *value_ptr == NULL) { - value_ptr++; - i++; - } - if (i >= n) + if (i >= d->ma_used) goto fail; key = DK_ENTRIES(k)[i].me_key; + assert(d->ma_values[i] != NULL); } else { + Py_ssize_t n = k->dk_nentries; PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; @@ -3461,7 +3459,7 @@ static PyObject * dictiter_iternextvalue(dictiterobject *di) { PyObject *value; - Py_ssize_t i, n; + Py_ssize_t i; PyDictObject *d = di->di_dict; if (d == NULL) @@ -3476,18 +3474,15 @@ dictiter_iternextvalue(dictiterobject *di) } i = di->di_pos; - n = d->ma_keys->dk_nentries; + assert(i >= 0); if (d->ma_values) { - PyObject **value_ptr = &d->ma_values[i]; - while (i < n && *value_ptr == NULL) { - value_ptr++; - i++; - } - if (i >= n) + if (i >= d->ma_used) goto fail; - value = *value_ptr; + value = d->ma_values[i]; + assert(value != NULL); } else { + Py_ssize_t n = d->ma_keys->dk_nentries; PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; @@ -3545,7 +3540,7 @@ static PyObject * dictiter_iternextitem(dictiterobject *di) { PyObject *key, *value, *result = di->di_result; - Py_ssize_t i, n; + Py_ssize_t i; PyDictObject *d = di->di_dict; if (d == NULL) @@ -3560,19 +3555,16 @@ dictiter_iternextitem(dictiterobject *di) } i = di->di_pos; - n = d->ma_keys->dk_nentries; + assert(i >= 0); if (d->ma_values) { - PyObject **value_ptr = &d->ma_values[i]; - while (i < n && *value_ptr == NULL) { - value_ptr++; - i++; - } - if (i >= n) + if (i >= d->ma_used) goto fail; key = DK_ENTRIES(d->ma_keys)[i].me_key; - value = *value_ptr; + value = d->ma_values[i]; + assert(value != NULL); } else { + Py_ssize_t n = d->ma_keys->dk_nentries; PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145..57a786c022 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -631,19 +631,17 @@ ImportError_init(PyImportErrorObject *self, PyObject *args, PyObject *kwds) } Py_DECREF(empty_tuple); - if (name) { - Py_INCREF(name); - Py_XSETREF(self->name, name); - } - if (path) { - Py_INCREF(path); - Py_XSETREF(self->path, path); - } + Py_XINCREF(name); + Py_XSETREF(self->name, name); + + Py_XINCREF(path); + Py_XSETREF(self->path, path); + if (PyTuple_GET_SIZE(args) == 1) { msg = PyTuple_GET_ITEM(args, 0); Py_INCREF(msg); - Py_XSETREF(self->msg, msg); } + Py_XSETREF(self->msg, msg); return 0; } @@ -1822,18 +1820,10 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) Py_CLEAR(err->object); Py_CLEAR(err->reason); - if (!PyArg_ParseTuple(args, "O!O!nnO!", - &PyUnicode_Type, &err->encoding, - &PyUnicode_Type, &err->object, - &err->start, - &err->end, - &PyUnicode_Type, &err->reason)) { - err->encoding = err->object = err->reason = NULL; - return -1; - } - - if (PyUnicode_READY(err->object) < -1) { - err->encoding = NULL; + if (!PyArg_ParseTuple(args, "UUnnU", + &err->encoding, &err->object, + &err->start, &err->end, &err->reason)) { + err->encoding = err->object = err->reason = NULL; return -1; } @@ -1937,12 +1927,9 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) Py_CLEAR(ude->object); Py_CLEAR(ude->reason); - if (!PyArg_ParseTuple(args, "O!OnnO!", - &PyUnicode_Type, &ude->encoding, - &ude->object, - &ude->start, - &ude->end, - &PyUnicode_Type, &ude->reason)) { + if (!PyArg_ParseTuple(args, "UOnnU", + &ude->encoding, &ude->object, + &ude->start, &ude->end, &ude->reason)) { ude->encoding = ude->object = ude->reason = NULL; return -1; } @@ -2052,11 +2039,9 @@ UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, Py_CLEAR(self->object); Py_CLEAR(self->reason); - if (!PyArg_ParseTuple(args, "O!nnO!", - &PyUnicode_Type, &self->object, - &self->start, - &self->end, - &PyUnicode_Type, &self->reason)) { + if (!PyArg_ParseTuple(args, "UnnU", + &self->object, + &self->start, &self->end, &self->reason)) { self->object = self->reason = NULL; return -1; } diff --git a/Objects/longobject.c b/Objects/longobject.c index a453241b3f..10dde726a2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -315,7 +315,6 @@ PyLong_FromUnsignedLong(unsigned long ival) v = _PyLong_New(ndigits); if (v != NULL) { digit *p = v->ob_digit; - Py_SIZE(v) = ndigits; while (ival) { *p++ = (digit)(ival & PyLong_MASK); ival >>= PyLong_SHIFT; @@ -709,12 +708,33 @@ _PyLong_Sign(PyObject *vv) return Py_SIZE(v) == 0 ? 0 : (Py_SIZE(v) < 0 ? -1 : 1); } +/* bits_in_digit(d) returns the unique integer k such that 2**(k-1) <= d < + 2**k if d is nonzero, else 0. */ + +static const unsigned char BitLengthTable[32] = { + 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 +}; + +static int +bits_in_digit(digit d) +{ + int d_bits = 0; + while (d >= 32) { + d_bits += 6; + d >>= 6; + } + d_bits += (int)BitLengthTable[d]; + return d_bits; +} + size_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; size_t result = 0; Py_ssize_t ndigits; + int msd_bits; assert(v != NULL); assert(PyLong_Check(v)); @@ -725,12 +745,10 @@ _PyLong_NumBits(PyObject *vv) if ((size_t)(ndigits - 1) > SIZE_MAX / (size_t)PyLong_SHIFT) goto Overflow; result = (size_t)(ndigits - 1) * (size_t)PyLong_SHIFT; - do { - ++result; - if (result == 0) - goto Overflow; - msd >>= 1; - } while (msd); + msd_bits = bits_in_digit(msd); + if (SIZE_MAX - msd_bits < result) + goto Overflow; + result += msd_bits; } return result; @@ -1102,7 +1120,6 @@ PyLong_FromUnsignedLongLong(unsigned long long ival) v = _PyLong_New(ndigits); if (v != NULL) { digit *p = v->ob_digit; - Py_SIZE(v) = ndigits; while (ival) { *p++ = (digit)(ival & PyLong_MASK); ival >>= PyLong_SHIFT; @@ -1416,26 +1433,6 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) Py_RETURN_NOTIMPLEMENTED; \ } while(0) -/* bits_in_digit(d) returns the unique integer k such that 2**(k-1) <= d < - 2**k if d is nonzero, else 0. */ - -static const unsigned char BitLengthTable[32] = { - 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 -}; - -static int -bits_in_digit(digit d) -{ - int d_bits = 0; - while (d >= 32) { - d_bits += 6; - d >>= 6; - } - d_bits += (int)BitLengthTable[d]; - return d_bits; -} - /* x[0:m] and y[0:n] are digit vectors, LSD first, m >= n required. x[0:n] * is modified in place, by adding y to it. Carries are propagated as far as * x[m-1], and the remaining carry (0 or 1) is returned. @@ -3105,9 +3102,7 @@ long_add(PyLongObject *a, PyLongObject *b) CHECK_BINOP(a, b); if (Py_ABS(Py_SIZE(a)) <= 1 && Py_ABS(Py_SIZE(b)) <= 1) { - PyObject *result = PyLong_FromLong(MEDIUM_VALUE(a) + - MEDIUM_VALUE(b)); - return result; + return PyLong_FromLong(MEDIUM_VALUE(a) + MEDIUM_VALUE(b)); } if (Py_SIZE(a) < 0) { if (Py_SIZE(b) < 0) { @@ -3141,9 +3136,7 @@ long_sub(PyLongObject *a, PyLongObject *b) CHECK_BINOP(a, b); if (Py_ABS(Py_SIZE(a)) <= 1 && Py_ABS(Py_SIZE(b)) <= 1) { - PyObject* r; - r = PyLong_FromLong(MEDIUM_VALUE(a)-MEDIUM_VALUE(b)); - return r; + return PyLong_FromLong(MEDIUM_VALUE(a) - MEDIUM_VALUE(b)); } if (Py_SIZE(a) < 0) { if (Py_SIZE(b) < 0) @@ -4296,22 +4289,22 @@ long_rshift(PyLongObject *a, PyLongObject *b) PyLongObject *a1, *a2; a1 = (PyLongObject *) long_invert(a); if (a1 == NULL) - goto rshift_error; + return NULL; a2 = (PyLongObject *) long_rshift(a1, b); Py_DECREF(a1); if (a2 == NULL) - goto rshift_error; + return NULL; z = (PyLongObject *) long_invert(a2); Py_DECREF(a2); } else { shiftby = PyLong_AsSsize_t((PyObject *)b); if (shiftby == -1L && PyErr_Occurred()) - goto rshift_error; + return NULL; if (shiftby < 0) { PyErr_SetString(PyExc_ValueError, "negative shift count"); - goto rshift_error; + return NULL; } wordshift = shiftby / PyLong_SHIFT; newsize = Py_ABS(Py_SIZE(a)) - wordshift; @@ -4323,19 +4316,15 @@ long_rshift(PyLongObject *a, PyLongObject *b) himask = PyLong_MASK ^ lomask; z = _PyLong_New(newsize); if (z == NULL) - goto rshift_error; - if (Py_SIZE(a) < 0) - Py_SIZE(z) = -(Py_SIZE(z)); + return NULL; for (i = 0, j = wordshift; i < newsize; i++, j++) { z->ob_digit[i] = (a->ob_digit[j] >> loshift) & lomask; if (i+1 < newsize) z->ob_digit[i] |= (a->ob_digit[j+1] << hishift) & himask; } - z = long_normalize(z); + z = maybe_small_long(long_normalize(z)); } - rshift_error: - return (PyObject *) maybe_small_long(z); - + return (PyObject *)z; } static PyObject * @@ -5089,7 +5078,8 @@ static PyObject * long_bit_length(PyLongObject *v) { PyLongObject *result, *x, *y; - Py_ssize_t ndigits, msd_bits = 0; + Py_ssize_t ndigits; + int msd_bits; digit msd; assert(v != NULL); @@ -5100,11 +5090,7 @@ long_bit_length(PyLongObject *v) return PyLong_FromLong(0); msd = v->ob_digit[ndigits-1]; - while (msd >= 32) { - msd_bits += 6; - msd >>= 6; - } - msd_bits += (long)(BitLengthTable[msd]); + msd_bits = bits_in_digit(msd); if (ndigits <= PY_SSIZE_T_MAX/PyLong_SHIFT) return PyLong_FromSsize_t((ndigits-1)*PyLong_SHIFT + msd_bits); diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24..45c557ff57 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -829,8 +829,6 @@ static PyMethodDef rangeiter_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyObject *rangeiter_new(PyTypeObject *, PyObject *args, PyObject *kw); - PyTypeObject PyRangeIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "range_iterator", /* tp_name */ @@ -862,15 +860,6 @@ PyTypeObject PyRangeIter_Type = { (iternextfunc)rangeiter_next, /* tp_iternext */ rangeiter_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - rangeiter_new, /* tp_new */ }; /* Return number of items in range (lo, hi, step). step != 0 @@ -925,36 +914,6 @@ fast_range_iter(long start, long stop, long step) return (PyObject *)it; } -static PyObject * -rangeiter_new(PyTypeObject *type, PyObject *args, PyObject *kw) -{ - long start, stop, step; - - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "range_iterator(): creating instances of range_iterator " - "by calling range_iterator type is deprecated", - 1)) { - return NULL; - } - - if (!_PyArg_NoKeywords("range_iterator()", kw)) { - return NULL; - } - - if (!PyArg_ParseTuple(args, - "lll;range_iterator() requires 3 int arguments", - &start, &stop, &step)) { - return NULL; - } - if (step == 0) { - PyErr_SetString(PyExc_ValueError, - "range_iterator() arg 3 must not be zero"); - return NULL; - } - - return fast_range_iter(start, stop, step); -} - typedef struct { PyObject_HEAD PyObject *index; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 249cf578d9..e22eed8853 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1029,8 +1029,7 @@ resize_copy(PyObject *unicode, Py_ssize_t length) if (_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND) { PyObject *copy; - if (PyUnicode_READY(unicode) == -1) - return NULL; + assert(PyUnicode_IS_READY(unicode)); copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode)); if (copy == NULL) @@ -1974,14 +1973,11 @@ unicode_char(Py_UCS4 ch) unicode = PyUnicode_New(1, ch); if (unicode == NULL) return NULL; - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - PyUnicode_1BYTE_DATA(unicode)[0] = (Py_UCS1)ch; - break; - case PyUnicode_2BYTE_KIND: + + assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND); + if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; - break; - default: + } else { assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); PyUnicode_4BYTE_DATA(unicode)[0] = ch; } @@ -3412,11 +3408,9 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) { Py_ssize_t wlen, wlen2; wchar_t *wstr; - PyObject *bytes = NULL; char *errmsg; - PyObject *reason = NULL; - PyObject *exc; - size_t error_pos; + PyObject *bytes, *reason, *exc; + size_t error_pos, errlen; int surrogateescape; if (locale_error_handler(errors, &surrogateescape) < 0) @@ -3471,6 +3465,7 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) len2 = wcstombs(PyBytes_AS_STRING(bytes), wstr, len+1); if (len2 == (size_t)-1 || len2 > len) { + Py_DECREF(bytes); error_pos = (size_t)-1; goto encode_error; } @@ -3486,17 +3481,15 @@ encode_error: error_pos = wcstombs_errorpos(wstr); PyMem_Free(wstr); - Py_XDECREF(bytes); - - if (errmsg != NULL) { - size_t errlen; - wstr = Py_DecodeLocale(errmsg, &errlen); - if (wstr != NULL) { - reason = PyUnicode_FromWideChar(wstr, errlen); - PyMem_RawFree(wstr); - } else - errmsg = NULL; + + wstr = Py_DecodeLocale(errmsg, &errlen); + if (wstr != NULL) { + reason = PyUnicode_FromWideChar(wstr, errlen); + PyMem_RawFree(wstr); + } else { + errmsg = NULL; } + if (errmsg == NULL) reason = PyUnicode_FromString( "wcstombs() encountered an unencodable " @@ -3512,7 +3505,7 @@ encode_error: Py_DECREF(reason); if (exc != NULL) { PyCodec_StrictErrors(exc); - Py_XDECREF(exc); + Py_DECREF(exc); } return NULL; } @@ -3719,10 +3712,9 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, size_t wlen, wlen2; PyObject *unicode; int surrogateescape; - size_t error_pos; + size_t error_pos, errlen; char *errmsg; - PyObject *reason = NULL; /* initialize to prevent gcc warning */ - PyObject *exc; + PyObject *exc, *reason = NULL; /* initialize to prevent gcc warning */ if (locale_error_handler(errors, &surrogateescape) < 0) return NULL; @@ -3780,19 +3772,16 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, return unicode; decode_error: - reason = NULL; errmsg = strerror(errno); assert(errmsg != NULL); error_pos = mbstowcs_errorpos(str, len); - if (errmsg != NULL) { - size_t errlen; - wstr = Py_DecodeLocale(errmsg, &errlen); - if (wstr != NULL) { - reason = PyUnicode_FromWideChar(wstr, errlen); - PyMem_RawFree(wstr); - } + wstr = Py_DecodeLocale(errmsg, &errlen); + if (wstr != NULL) { + reason = PyUnicode_FromWideChar(wstr, errlen); + PyMem_RawFree(wstr); } + if (reason == NULL) reason = PyUnicode_FromString( "mbstowcs() encountered an invalid multibyte sequence"); @@ -3807,7 +3796,7 @@ decode_error: Py_DECREF(reason); if (exc != NULL) { PyCodec_StrictErrors(exc); - Py_XDECREF(exc); + Py_DECREF(exc); } return NULL; } @@ -4248,7 +4237,7 @@ unicode_decode_call_errorhandler_wchar( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, PyObject **output, Py_ssize_t *outpos) { - static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "Un;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4281,10 +4270,10 @@ unicode_decode_call_errorhandler_wchar( if (restuple == NULL) goto onError; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[4]); + PyErr_SetString(PyExc_TypeError, &argparse[3]); goto onError; } - if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos)) + if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos)) goto onError; /* Copy back the bytes variables, which might have been modified by the @@ -4292,9 +4281,6 @@ unicode_decode_call_errorhandler_wchar( inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject); if (!inputobj) goto onError; - if (!PyBytes_Check(inputobj)) { - PyErr_Format(PyExc_TypeError, "exception attribute object must be bytes"); - } *input = PyBytes_AS_STRING(inputobj); insize = PyBytes_GET_SIZE(inputobj); *inend = *input + insize; @@ -4335,7 +4321,7 @@ unicode_decode_call_errorhandler_wchar( *inptr = *input + newpos; /* we made it! */ - Py_XDECREF(restuple); + Py_DECREF(restuple); return 0; overflow: @@ -4356,7 +4342,7 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */) { - static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "Un;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4383,10 +4369,10 @@ unicode_decode_call_errorhandler_writer( if (restuple == NULL) goto onError; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[4]); + PyErr_SetString(PyExc_TypeError, &argparse[3]); goto onError; } - if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos)) + if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos)) goto onError; /* Copy back the bytes variables, which might have been modified by the @@ -4394,9 +4380,6 @@ unicode_decode_call_errorhandler_writer( inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject); if (!inputobj) goto onError; - if (!PyBytes_Check(inputobj)) { - PyErr_Format(PyExc_TypeError, "exception attribute object must be bytes"); - } *input = PyBytes_AS_STRING(inputobj); insize = PyBytes_GET_SIZE(inputobj); *inend = *input + insize; @@ -4411,8 +4394,6 @@ unicode_decode_call_errorhandler_writer( goto onError; } - if (PyUnicode_READY(repunicode) < 0) - goto onError; replen = PyUnicode_GET_LENGTH(repunicode); if (replen > 1) { writer->min_length += replen - 1; @@ -4428,7 +4409,7 @@ unicode_decode_call_errorhandler_writer( *inptr = *input + newpos; /* we made it! */ - Py_XDECREF(restuple); + Py_DECREF(restuple); return 0; onError: @@ -8663,7 +8644,7 @@ unicode_translate_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static const char *argparse = "O!n;translating error handler must return (str, int) tuple"; + static const char *argparse = "Un;translating error handler must return (str, int) tuple"; Py_ssize_t i_newpos; PyObject *restuple; @@ -8685,11 +8666,11 @@ unicode_translate_call_errorhandler(const char *errors, if (restuple == NULL) return NULL; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[4]); + PyErr_SetString(PyExc_TypeError, &argparse[3]); Py_DECREF(restuple); return NULL; } - if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, + if (!PyArg_ParseTuple(restuple, argparse, &resunicode, &i_newpos)) { Py_DECREF(restuple); return NULL; |