diff options
Diffstat (limited to 'Objects/stringlib')
| -rw-r--r-- | Objects/stringlib/asciilib.h | 30 | ||||
| -rw-r--r-- | Objects/stringlib/codecs.h | 629 | ||||
| -rw-r--r-- | Objects/stringlib/count.h | 9 | ||||
| -rw-r--r-- | Objects/stringlib/eq.h | 23 | ||||
| -rw-r--r-- | Objects/stringlib/fastsearch.h | 75 | ||||
| -rw-r--r-- | Objects/stringlib/find.h | 89 | ||||
| -rw-r--r-- | Objects/stringlib/find_max_char.h | 133 | ||||
| -rw-r--r-- | Objects/stringlib/formatter.h | 1518 | ||||
| -rw-r--r-- | Objects/stringlib/localeutil.h | 100 | ||||
| -rw-r--r-- | Objects/stringlib/partition.h | 12 | ||||
| -rw-r--r-- | Objects/stringlib/split.h | 26 | ||||
| -rw-r--r-- | Objects/stringlib/stringdefs.h | 8 | ||||
| -rw-r--r-- | Objects/stringlib/ucs1lib.h | 31 | ||||
| -rw-r--r-- | Objects/stringlib/ucs2lib.h | 30 | ||||
| -rw-r--r-- | Objects/stringlib/ucs4lib.h | 30 | ||||
| -rw-r--r-- | Objects/stringlib/undef.h | 12 | ||||
| -rw-r--r-- | Objects/stringlib/unicode_format.h (renamed from Objects/stringlib/string_format.h) | 444 | ||||
| -rw-r--r-- | Objects/stringlib/unicodedefs.h | 8 |
18 files changed, 1288 insertions, 1919 deletions
diff --git a/Objects/stringlib/asciilib.h b/Objects/stringlib/asciilib.h new file mode 100644 index 0000000000..f62813d2fd --- /dev/null +++ b/Objects/stringlib/asciilib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH asciilib_fastsearch +#define STRINGLIB(F) asciilib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 1 +#define STRINGLIB_MAX_CHAR 0x7Fu +#define STRINGLIB_CHAR Py_UCS1 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((char*)(STR),(LEN)) +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ascii_InsertThousandsGrouping + diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h new file mode 100644 index 0000000000..2a01089c0f --- /dev/null +++ b/Objects/stringlib/codecs.h @@ -0,0 +1,629 @@ +/* stringlib: codec implementations */ + +#if STRINGLIB_IS_UNICODE + +/* Mask to quickly check whether a C 'long' contains a + non-ASCII, UTF8-encoded char. */ +#if (SIZEOF_LONG == 8) +# define ASCII_CHAR_MASK 0x8080808080808080UL +#elif (SIZEOF_LONG == 4) +# define ASCII_CHAR_MASK 0x80808080UL +#else +# error C 'long' size should be either 4 or 8! +#endif + +/* 10xxxxxx */ +#define IS_CONTINUATION_BYTE(ch) ((ch) >= 0x80 && (ch) < 0xC0) + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(utf8_decode)(const char **inptr, const char *end, + STRINGLIB_CHAR *dest, + Py_ssize_t *outpos) +{ + Py_UCS4 ch; + const char *s = *inptr; + const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); + STRINGLIB_CHAR *p = dest + *outpos; + + while (s < end) { + ch = (unsigned char)*s; + + if (ch < 0x80) { + /* Fast path for runs of ASCII characters. Given that common UTF-8 + input will consist of an overwhelming majority of ASCII + characters, we try to optimize for this case by checking + as many characters as a C 'long' can contain. + First, check if we can do an aligned read, as most CPUs have + a penalty for unaligned reads. + */ + if (_Py_IS_ALIGNED(s, SIZEOF_LONG)) { + /* Help register allocation */ + register const char *_s = s; + register STRINGLIB_CHAR *_p = p; + while (_s < aligned_end) { + /* Read a whole long at a time (either 4 or 8 bytes), + and do a fast unrolled copy if it only contains ASCII + characters. */ + unsigned long value = *(unsigned long *) _s; + if (value & ASCII_CHAR_MASK) + break; +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + _p[0] = (STRINGLIB_CHAR)(value & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); +# if SIZEOF_LONG == 8 + _p[4] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); +# endif +#else +# if SIZEOF_LONG == 8 + _p[0] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[4] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)(value & 0xFFu); +# else + _p[0] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)(value & 0xFFu); +# endif +#endif + _s += SIZEOF_LONG; + _p += SIZEOF_LONG; + } + s = _s; + p = _p; + if (s == end) + break; + ch = (unsigned char)*s; + } + if (ch < 0x80) { + s++; + *p++ = ch; + continue; + } + } + + if (ch < 0xC2) { + /* invalid sequence + \x80-\xBF -- continuation byte + \xC0-\xC1 -- fake 0000-007F */ + goto InvalidStart; + } + + if (ch < 0xE0) { + /* \xC2\x80-\xDF\xBF -- 0080-07FF */ + Py_UCS4 ch2; + if (end - s < 2) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + if (!IS_CONTINUATION_BYTE(ch2)) + /* invalid continuation byte */ + goto InvalidContinuation; + ch = (ch << 6) + ch2 - + ((0xC0 << 6) + 0x80); + assert ((ch > 0x007F) && (ch <= 0x07FF)); + s += 2; + if (STRINGLIB_MAX_CHAR <= 0x007F || + (STRINGLIB_MAX_CHAR < 0x07FF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + + if (ch < 0xF0) { + /* \xE0\xA0\x80-\xEF\xBF\xBF -- 0800-FFFF */ + Py_UCS4 ch2, ch3; + if (end - s < 3) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + ch3 = (unsigned char)s[2]; + if (!IS_CONTINUATION_BYTE(ch2) || + !IS_CONTINUATION_BYTE(ch3)) { + /* invalid continuation byte */ + goto InvalidContinuation; + } + if (ch == 0xE0) { + if (ch2 < 0xA0) + /* invalid sequence + \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */ + goto InvalidContinuation; + } + else if (ch == 0xED && ch2 > 0x9F) { + /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF + will result in surrogates in range D800-DFFF. Surrogates are + not valid UTF-8 so they are rejected. + See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf + (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ + goto InvalidContinuation; + } + ch = (ch << 12) + (ch2 << 6) + ch3 - + ((0xE0 << 12) + (0x80 << 6) + 0x80); + assert ((ch > 0x07FF) && (ch <= 0xFFFF)); + s += 3; + if (STRINGLIB_MAX_CHAR <= 0x07FF || + (STRINGLIB_MAX_CHAR < 0xFFFF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + + if (ch < 0xF5) { + /* \xF0\x90\x80\x80-\xF4\x8F\xBF\xBF -- 10000-10FFFF */ + Py_UCS4 ch2, ch3, ch4; + if (end - s < 4) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + ch3 = (unsigned char)s[2]; + ch4 = (unsigned char)s[3]; + if (!IS_CONTINUATION_BYTE(ch2) || + !IS_CONTINUATION_BYTE(ch3) || + !IS_CONTINUATION_BYTE(ch4)) { + /* invalid continuation byte */ + goto InvalidContinuation; + } + if (ch == 0xF0) { + if (ch2 < 0x90) + /* invalid sequence + \xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */ + goto InvalidContinuation; + } + else if (ch == 0xF4 && ch2 > 0x8F) { + /* invalid sequence + \xF4\x90\x80\80- -- 110000- overflow */ + goto InvalidContinuation; + } + ch = (ch << 18) + (ch2 << 12) + (ch3 << 6) + ch4 - + ((0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80); + assert ((ch > 0xFFFF) && (ch <= 0x10FFFF)); + s += 4; + if (STRINGLIB_MAX_CHAR <= 0xFFFF || + (STRINGLIB_MAX_CHAR < 0x10FFFF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + goto InvalidStart; + } + ch = 0; +Overflow: +Return: + *inptr = s; + *outpos = p - dest; + return ch; +InvalidStart: + ch = 1; + goto Return; +InvalidContinuation: + ch = 2; + goto Return; +} + +#undef ASCII_CHAR_MASK +#undef IS_CONTINUATION_BYTE + + +/* UTF-8 encoder specialized for a Unicode kind to avoid the slow + PyUnicode_READ() macro. Delete some parts of the code depending on the kind: + UCS-1 strings don't need to handle surrogates for example. */ +Py_LOCAL_INLINE(PyObject *) +STRINGLIB(utf8_encoder)(PyObject *unicode, + STRINGLIB_CHAR *data, + Py_ssize_t size, + const char *errors) +{ +#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ + + Py_ssize_t i; /* index into s of next input byte */ + PyObject *result; /* result string object */ + char *p; /* next free byte in output buffer */ + Py_ssize_t nallocated; /* number of result bytes allocated */ + Py_ssize_t nneeded; /* number of result bytes needed */ +#if STRINGLIB_SIZEOF_CHAR > 1 + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; +#endif +#if STRINGLIB_SIZEOF_CHAR == 1 + const Py_ssize_t max_char_size = 2; + char stackbuf[MAX_SHORT_UNICHARS * 2]; +#elif STRINGLIB_SIZEOF_CHAR == 2 + const Py_ssize_t max_char_size = 3; + char stackbuf[MAX_SHORT_UNICHARS * 3]; +#else /* STRINGLIB_SIZEOF_CHAR == 4 */ + const Py_ssize_t max_char_size = 4; + char stackbuf[MAX_SHORT_UNICHARS * 4]; +#endif + + assert(size >= 0); + + if (size <= MAX_SHORT_UNICHARS) { + /* Write into the stack buffer; nallocated can't overflow. + * At the end, we'll allocate exactly as much heap space as it + * turns out we need. + */ + nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); + result = NULL; /* will allocate after we're done */ + p = stackbuf; + } + else { + if (size > PY_SSIZE_T_MAX / max_char_size) { + /* integer overflow */ + return PyErr_NoMemory(); + } + /* Overallocate on the heap, and give the excess back at the end. */ + nallocated = size * max_char_size; + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + return NULL; + p = PyBytes_AS_STRING(result); + } + + for (i = 0; i < size;) { + Py_UCS4 ch = data[i++]; + + if (ch < 0x80) { + /* Encode ASCII */ + *p++ = (char) ch; + + } + else +#if STRINGLIB_SIZEOF_CHAR > 1 + if (ch < 0x0800) +#endif + { + /* Encode Latin-1 */ + *p++ = (char)(0xc0 | (ch >> 6)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#if STRINGLIB_SIZEOF_CHAR > 1 + else if (Py_UNICODE_IS_SURROGATE(ch)) { + Py_ssize_t newpos; + Py_ssize_t repsize, k, startpos; + startpos = i-1; + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, "utf-8", "surrogates not allowed", + unicode, &exc, startpos, startpos+1, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_LENGTH(rep); + + if (repsize > max_char_size) { + Py_ssize_t offset; + + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { + /* integer overflow */ + PyErr_NoMemory(); + goto error; + } + nallocated += repsize - max_char_size; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; + } + + if (PyBytes_Check(rep)) { + char *prep = PyBytes_AS_STRING(rep); + for(k = repsize; k > 0; k--) + *p++ = *prep++; + } else /* rep is unicode */ { + enum PyUnicode_Kind repkind; + void *repdata; + + if (PyUnicode_READY(rep) < 0) + goto error; + repkind = PyUnicode_KIND(rep); + repdata = PyUnicode_DATA(rep); + + for(k=0; k<repsize; k++) { + Py_UCS4 c = PyUnicode_READ(repkind, repdata, k); + if (0x80 <= c) { + raise_encode_exception(&exc, "utf-8", + unicode, + i-1, i, + "surrogates not allowed"); + goto error; + } + *p++ = (char)c; + } + } + Py_CLEAR(rep); + } + else +#if STRINGLIB_SIZEOF_CHAR > 2 + if (ch < 0x10000) +#endif + { + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#if STRINGLIB_SIZEOF_CHAR > 2 + else /* ch >= 0x10000 */ + { + assert(ch <= MAX_UNICODE); + /* Encode UCS4 Unicode ordinals */ + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#endif /* STRINGLIB_SIZEOF_CHAR > 2 */ +#endif /* STRINGLIB_SIZEOF_CHAR > 1 */ + } + + if (result == NULL) { + /* This was stack allocated. */ + nneeded = p - stackbuf; + assert(nneeded <= nallocated); + result = PyBytes_FromStringAndSize(stackbuf, nneeded); + } + else { + /* Cut back to size actually needed. */ + nneeded = p - PyBytes_AS_STRING(result); + assert(nneeded <= nallocated); + _PyBytes_Resize(&result, nneeded); + } + +#if STRINGLIB_SIZEOF_CHAR > 1 + Py_XDECREF(errorHandler); + Py_XDECREF(exc); +#endif + return result; + +#if STRINGLIB_SIZEOF_CHAR > 1 + error: + Py_XDECREF(rep); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + Py_XDECREF(result); + return NULL; +#endif + +#undef MAX_SHORT_UNICHARS +} + +/* The pattern for constructing UCS2-repeated masks. */ +#if SIZEOF_LONG == 8 +# define UCS2_REPEAT_MASK 0x0001000100010001ul +#elif SIZEOF_LONG == 4 +# define UCS2_REPEAT_MASK 0x00010001ul +#else +# error C 'long' size should be either 4 or 8! +#endif + +/* The mask for fast checking. */ +#if STRINGLIB_SIZEOF_CHAR == 1 +/* The mask for fast checking of whether a C 'long' contains a + non-ASCII or non-Latin1 UTF16-encoded characters. */ +# define FAST_CHAR_MASK (UCS2_REPEAT_MASK * (0xFFFFu & ~STRINGLIB_MAX_CHAR)) +#else +/* The mask for fast checking of whether a C 'long' may contain + UTF16-encoded surrogate characters. This is an efficient heuristic, + assuming that non-surrogate characters with a code point >= 0x8000 are + rare in most input. +*/ +# define FAST_CHAR_MASK (UCS2_REPEAT_MASK * 0x8000u) +#endif +/* The mask for fast byte-swapping. */ +#define STRIPPED_MASK (UCS2_REPEAT_MASK * 0x00FFu) +/* Swap bytes. */ +#define SWAB(value) ((((value) >> 8) & STRIPPED_MASK) | \ + (((value) & STRIPPED_MASK) << 8)) + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, + STRINGLIB_CHAR *dest, Py_ssize_t *outpos, + int native_ordering) +{ + Py_UCS4 ch; + const unsigned char *aligned_end = + (const unsigned char *) _Py_ALIGN_DOWN(e, SIZEOF_LONG); + const unsigned char *q = *inptr; + STRINGLIB_CHAR *p = dest + *outpos; + /* Offsets from q for retrieving byte pairs in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int ihi = !!native_ordering, ilo = !native_ordering; +#else + int ihi = !native_ordering, ilo = !!native_ordering; +#endif + --e; + + while (q < e) { + Py_UCS4 ch2; + /* First check for possible aligned read of a C 'long'. Unaligned + reads are more expensive, better to defer to another iteration. */ + if (_Py_IS_ALIGNED(q, SIZEOF_LONG)) { + /* Fast path for runs of in-range non-surrogate chars. */ + register const unsigned char *_q = q; + while (_q < aligned_end) { + unsigned long block = * (unsigned long *) _q; + if (native_ordering) { + /* Can use buffer directly */ + if (block & FAST_CHAR_MASK) + break; + } + else { + /* Need to byte-swap */ + if (block & SWAB(FAST_CHAR_MASK)) + break; +#if STRINGLIB_SIZEOF_CHAR == 1 + block >>= 8; +#else + block = SWAB(block); +#endif + } +#ifdef BYTEORDER_IS_LITTLE_ENDIAN +# if SIZEOF_LONG == 4 + p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu); + p[1] = (STRINGLIB_CHAR)(block >> 16); +# elif SIZEOF_LONG == 8 + p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu); + p[1] = (STRINGLIB_CHAR)((block >> 16) & 0xFFFFu); + p[2] = (STRINGLIB_CHAR)((block >> 32) & 0xFFFFu); + p[3] = (STRINGLIB_CHAR)(block >> 48); +# endif +#else +# if SIZEOF_LONG == 4 + p[0] = (STRINGLIB_CHAR)(block >> 16); + p[1] = (STRINGLIB_CHAR)(block & 0xFFFFu); +# elif SIZEOF_LONG == 8 + p[0] = (STRINGLIB_CHAR)(block >> 48); + p[1] = (STRINGLIB_CHAR)((block >> 32) & 0xFFFFu); + p[2] = (STRINGLIB_CHAR)((block >> 16) & 0xFFFFu); + p[3] = (STRINGLIB_CHAR)(block & 0xFFFFu); +# endif +#endif + _q += SIZEOF_LONG; + p += SIZEOF_LONG / 2; + } + q = _q; + if (q >= e) + break; + } + + ch = (q[ihi] << 8) | q[ilo]; + q += 2; + if (!Py_UNICODE_IS_SURROGATE(ch)) { +#if STRINGLIB_SIZEOF_CHAR < 2 + if (ch > STRINGLIB_MAX_CHAR) + /* Out-of-range */ + goto Return; +#endif + *p++ = (STRINGLIB_CHAR)ch; + continue; + } + + /* UTF-16 code pair: */ + if (q >= e) + goto UnexpectedEnd; + if (!Py_UNICODE_IS_HIGH_SURROGATE(ch)) + goto IllegalEncoding; + ch2 = (q[ihi] << 8) | q[ilo]; + q += 2; + if (!Py_UNICODE_IS_LOW_SURROGATE(ch2)) + goto IllegalSurrogate; + ch = Py_UNICODE_JOIN_SURROGATES(ch, ch2); +#if STRINGLIB_SIZEOF_CHAR < 4 + /* Out-of-range */ + goto Return; +#else + *p++ = (STRINGLIB_CHAR)ch; +#endif + } + ch = 0; +Return: + *inptr = q; + *outpos = p - dest; + return ch; +UnexpectedEnd: + ch = 1; + goto Return; +IllegalEncoding: + ch = 2; + goto Return; +IllegalSurrogate: + ch = 3; + goto Return; +} +#undef UCS2_REPEAT_MASK +#undef FAST_CHAR_MASK +#undef STRIPPED_MASK +#undef SWAB + + +Py_LOCAL_INLINE(void) +STRINGLIB(utf16_encode)(unsigned short *out, + const STRINGLIB_CHAR *in, + Py_ssize_t len, + int native_ordering) +{ + const STRINGLIB_CHAR *end = in + len; +#if STRINGLIB_SIZEOF_CHAR == 1 +# define SWAB2(CH) ((CH) << 8) +#else +# define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) +#endif +#if STRINGLIB_MAX_CHAR < 0x10000 + if (native_ordering) { +# if STRINGLIB_SIZEOF_CHAR == 2 + Py_MEMCPY(out, in, 2 * len); +# else + _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); +# endif + } else { + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } + while (in < end) { + *out++ = SWAB2(*in); + ++in; + } + } +#else + if (native_ordering) { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = ch; + else { + out[0] = Py_UNICODE_HIGH_SURROGATE(ch); + out[1] = Py_UNICODE_LOW_SURROGATE(ch); + out += 2; + } + } + } else { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = SWAB2((Py_UCS2)ch); + else { + Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); + Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); + out[0] = SWAB2(ch1); + out[1] = SWAB2(ch2); + out += 2; + } + } + } +#endif +#undef SWAB2 +} +#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/count.h b/Objects/stringlib/count.h index de34f96b3e..f48500bf56 100644 --- a/Objects/stringlib/count.h +++ b/Objects/stringlib/count.h @@ -1,14 +1,11 @@ /* stringlib: count implementation */ -#ifndef STRINGLIB_COUNT_H -#define STRINGLIB_COUNT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(count)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t maxcount) { @@ -19,7 +16,7 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return (str_len < maxcount) ? str_len + 1 : maxcount; - count = fastsearch(str, str_len, sub, sub_len, maxcount, FAST_COUNT); + count = FASTSEARCH(str, str_len, sub, sub_len, maxcount, FAST_COUNT); if (count < 0) return 0; /* no match */ @@ -27,4 +24,4 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, return count; } -#endif + diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h index 3e7f5e86c6..8e79a43f72 100644 --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -9,13 +9,26 @@ unicode_eq(PyObject *aa, PyObject *bb) register PyUnicodeObject *a = (PyUnicodeObject *)aa; register PyUnicodeObject *b = (PyUnicodeObject *)bb; - if (a->length != b->length) + if (PyUnicode_READY(a) == -1 || PyUnicode_READY(b) == -1) { + assert(0 && "unicode_eq ready fail"); return 0; - if (a->length == 0) + } + + if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + return 0; + if (PyUnicode_GET_LENGTH(a) == 0) return 1; - if (a->str[0] != b->str[0]) + if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) + return 0; + /* Just comparing the first byte is enough to see if a and b differ. + * If they are 2 byte or 4 byte character most differences will happen in + * the lower bytes anyways. + */ + if (PyUnicode_1BYTE_DATA(a)[0] != PyUnicode_1BYTE_DATA(b)[0]) return 0; - if (a->length == 1) + if (PyUnicode_KIND(a) == PyUnicode_1BYTE_KIND && + PyUnicode_GET_LENGTH(a) == 1) return 1; - return memcmp(a->str, b->str, a->length * sizeof(Py_UNICODE)) == 0; + return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), + PyUnicode_GET_LENGTH(a) * PyUnicode_KIND(a)) == 0; } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index e231c587e4..ecf885e7e1 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -1,6 +1,5 @@ /* stringlib: fastsearch implementation */ -#ifndef STRINGLIB_FASTSEARCH_H #define STRINGLIB_FASTSEARCH_H /* fast search/count implementation, based on a mix between boyer- @@ -33,8 +32,61 @@ #define STRINGLIB_BLOOM(mask, ch) \ ((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1))))) + +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(fastsearch_memchr_1char)(const STRINGLIB_CHAR* s, Py_ssize_t n, + STRINGLIB_CHAR ch, unsigned char needle, + Py_ssize_t maxcount, int mode) +{ + void *candidate; + const STRINGLIB_CHAR *found; + +#define DO_MEMCHR(memchr, s, needle, nchars) do { \ + candidate = memchr((const void *) (s), (needle), (nchars) * sizeof(STRINGLIB_CHAR)); \ + found = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); \ + } while (0) + + if (mode == FAST_SEARCH) { + const STRINGLIB_CHAR *ptr = s; + const STRINGLIB_CHAR *e = s + n; + while (ptr < e) { + DO_MEMCHR(memchr, ptr, needle, e - ptr); + if (found == NULL) + return -1; + if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) + return (found - s); + /* False positive */ + ptr = found + 1; + } + return -1; + } +#ifdef HAVE_MEMRCHR + /* memrchr() is a GNU extension, available since glibc 2.1.91. + it doesn't seem as optimized as memchr(), but is still quite + faster than our hand-written loop in FASTSEARCH below */ + else if (mode == FAST_RSEARCH) { + while (n > 0) { + DO_MEMCHR(memrchr, s, needle, n); + if (found == NULL) + return -1; + n = found - s; + if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) + return n; + /* False positive */ + } + return -1; + } +#endif + else { + assert(0); /* Should never get here */ + return 0; + } + +#undef DO_MEMCHR +} + Py_LOCAL_INLINE(Py_ssize_t) -fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, +FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, Py_ssize_t maxcount, int mode) { @@ -52,6 +104,24 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, if (m <= 0) return -1; /* use special case for 1-character strings */ + if (n > 10 && (mode == FAST_SEARCH +#ifdef HAVE_MEMRCHR + || mode == FAST_RSEARCH +#endif + )) { + /* use memchr if we can choose a needle without two many likely + false positives */ + unsigned char needle; + needle = p[0] & 0xff; +#if STRINGLIB_SIZEOF_CHAR > 1 + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) +#endif + return STRINGLIB(fastsearch_memchr_1char) + (s, n, p[0], needle, maxcount, mode); + } if (mode == FAST_COUNT) { for (i = 0; i < n; i++) if (s[i] == p[0]) { @@ -157,4 +227,3 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, return count; } -#endif diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index ce615dcb8a..518e012b75 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -1,14 +1,11 @@ /* stringlib: find/index implementation */ -#ifndef STRINGLIB_FIND_H -#define STRINGLIB_FIND_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -19,7 +16,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_SEARCH); if (pos >= 0) pos += offset; @@ -28,7 +25,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -39,7 +36,7 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return str_len + offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_RSEARCH); if (pos >= 0) pos += offset; @@ -63,29 +60,29 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_find(str + start, end - start, sub, sub_len, start); + return STRINGLIB(find)(str + start, end - start, sub, sub_len, start); } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_rfind(str + start, end - start, sub, sub_len, start); + return STRINGLIB(rfind)(str + start, end - start, sub, sub_len, start); } #ifdef STRINGLIB_WANT_CONTAINS_OBJ Py_LOCAL_INLINE(int) -stringlib_contains_obj(PyObject* str, PyObject* sub) +STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) { - return stringlib_find( + return STRINGLIB(find)( STRINGLIB_STR(str), STRINGLIB_LEN(str), STRINGLIB_STR(sub), STRINGLIB_LEN(sub), 0 ) != -1; @@ -98,14 +95,14 @@ This function is a helper for the "find" family (find, rfind, index, rindex) and for count, startswith and endswith, because they all have the same behaviour for the arguments. -It does not touch the variables received until it knows everything +It does not touch the variables received until it knows everything is ok. */ #define FORMAT_BUFFER_SIZE 50 Py_LOCAL_INLINE(int) -stringlib_parse_args_finds(const char * function_name, PyObject *args, +STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, PyObject **subobj, Py_ssize_t *start, Py_ssize_t *end) { @@ -148,28 +145,76 @@ first argument is a unicode object. Note that we receive a pointer to the pointer of the substring object, so when we create that object in this function we don't DECREF it, -because it continues living in the caller functions (those functions, +because it continues living in the caller functions (those functions, after finishing using the substring, must DECREF it). */ Py_LOCAL_INLINE(int) -stringlib_parse_args_finds_unicode(const char * function_name, PyObject *args, - PyUnicodeObject **substring, +STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, + PyObject **substring, Py_ssize_t *start, Py_ssize_t *end) { PyObject *tmp_substring; - if(stringlib_parse_args_finds(function_name, args, &tmp_substring, + if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring, start, end)) { tmp_substring = PyUnicode_FromObject(tmp_substring); if (!tmp_substring) return 0; - *substring = (PyUnicodeObject *)tmp_substring; + *substring = tmp_substring; return 1; } return 0; } -#endif /* STRINGLIB_IS_UNICODE */ +#else /* !STRINGLIB_IS_UNICODE */ + +/* +Wraps stringlib_parse_args_finds() and additionally checks whether the +first argument is an integer in range(0, 256). + +If this is the case, writes the integer value to the byte parameter +and sets subobj to NULL. Otherwise, sets the first argument to subobj +and doesn't touch byte. The other parameters are similar to those of +stringlib_parse_args_finds(). +*/ + +Py_LOCAL_INLINE(int) +STRINGLIB(parse_args_finds_byte)(const char *function_name, PyObject *args, + PyObject **subobj, char *byte, + Py_ssize_t *start, Py_ssize_t *end) +{ + PyObject *tmp_subobj; + Py_ssize_t ival; + PyObject *err; + + if(!STRINGLIB(parse_args_finds)(function_name, args, &tmp_subobj, + start, end)) + return 0; -#endif /* STRINGLIB_FIND_H */ + if (!PyNumber_Check(tmp_subobj)) { + *subobj = tmp_subobj; + return 1; + } + + ival = PyNumber_AsSsize_t(tmp_subobj, PyExc_OverflowError); + if (ival == -1) { + err = PyErr_Occurred(); + if (err && !PyErr_GivenExceptionMatches(err, PyExc_OverflowError)) { + PyErr_Clear(); + *subobj = tmp_subobj; + return 1; + } + } + + if (ival < 0 || ival > 255) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return 0; + } + + *subobj = NULL; + *byte = (char)ival; + return 1; +} + +#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h new file mode 100644 index 0000000000..06559c8a9f --- /dev/null +++ b/Objects/stringlib/find_max_char.h @@ -0,0 +1,133 @@ +/* Finding the optimal width of unicode characters in a buffer */ + +#if STRINGLIB_IS_UNICODE + +/* Mask to quickly check whether a C 'long' contains a + non-ASCII, UTF8-encoded char. */ +#if (SIZEOF_LONG == 8) +# define UCS1_ASCII_CHAR_MASK 0x8080808080808080UL +#elif (SIZEOF_LONG == 4) +# define UCS1_ASCII_CHAR_MASK 0x80808080UL +#else +# error C 'long' size should be either 4 or 8! +#endif + +#if STRINGLIB_SIZEOF_CHAR == 1 + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) +{ + const unsigned char *p = (const unsigned char *) begin; + const unsigned char *aligned_end = + (const unsigned char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); + + while (p < end) { + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { + /* Help register allocation */ + register const unsigned char *_p = p; + while (_p < aligned_end) { + unsigned long value = *(unsigned long *) _p; + if (value & UCS1_ASCII_CHAR_MASK) + return 255; + _p += SIZEOF_LONG; + } + p = _p; + if (p == end) + break; + } + if (*p++ & 0x80) + return 255; + } + return 127; +} + +#undef ASCII_CHAR_MASK + +#else /* STRINGLIB_SIZEOF_CHAR == 1 */ + +#define MASK_ASCII 0xFFFFFF80 +#define MASK_UCS1 0xFFFFFF00 +#define MASK_UCS2 0xFFFF0000 + +#define MAX_CHAR_ASCII 0x7f +#define MAX_CHAR_UCS1 0xff +#define MAX_CHAR_UCS2 0xffff +#define MAX_CHAR_UCS4 0x10ffff + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) +{ +#if STRINGLIB_SIZEOF_CHAR == 2 + const Py_UCS4 mask_limit = MASK_UCS1; + const Py_UCS4 max_char_limit = MAX_CHAR_UCS2; +#elif STRINGLIB_SIZEOF_CHAR == 4 + const Py_UCS4 mask_limit = MASK_UCS2; + const Py_UCS4 max_char_limit = MAX_CHAR_UCS4; +#else +#error Invalid STRINGLIB_SIZEOF_CHAR (must be 1, 2 or 4) +#endif + register Py_UCS4 mask; + Py_ssize_t n = end - begin; + const STRINGLIB_CHAR *p = begin; + const STRINGLIB_CHAR *unrolled_end = begin + _Py_SIZE_ROUND_DOWN(n, 4); + Py_UCS4 max_char; + + max_char = MAX_CHAR_ASCII; + mask = MASK_ASCII; + while (p < unrolled_end) { + STRINGLIB_CHAR bits = p[0] | p[1] | p[2] | p[3]; + if (bits & mask) { + if (mask == mask_limit) { + /* Limit reached */ + return max_char_limit; + } + if (mask == MASK_ASCII) { + max_char = MAX_CHAR_UCS1; + mask = MASK_UCS1; + } + else { + /* mask can't be MASK_UCS2 because of mask_limit above */ + assert(mask == MASK_UCS1); + max_char = MAX_CHAR_UCS2; + mask = MASK_UCS2; + } + /* We check the new mask on the same chars in the next iteration */ + continue; + } + p += 4; + } + while (p < end) { + if (p[0] & mask) { + if (mask == mask_limit) { + /* Limit reached */ + return max_char_limit; + } + if (mask == MASK_ASCII) { + max_char = MAX_CHAR_UCS1; + mask = MASK_UCS1; + } + else { + /* mask can't be MASK_UCS2 because of mask_limit above */ + assert(mask == MASK_UCS1); + max_char = MAX_CHAR_UCS2; + mask = MASK_UCS2; + } + /* We check the new mask on the same chars in the next iteration */ + continue; + } + p++; + } + return max_char; +} + +#undef MASK_ASCII +#undef MASK_UCS1 +#undef MASK_UCS2 +#undef MAX_CHAR_ASCII +#undef MAX_CHAR_UCS1 +#undef MAX_CHAR_UCS2 +#undef MAX_CHAR_UCS4 + +#endif /* STRINGLIB_SIZEOF_CHAR == 1 */ +#endif /* STRINGLIB_IS_UNICODE */ + diff --git a/Objects/stringlib/formatter.h b/Objects/stringlib/formatter.h deleted file mode 100644 index 4fdc62d650..0000000000 --- a/Objects/stringlib/formatter.h +++ /dev/null @@ -1,1518 +0,0 @@ -/* implements the string, long, and float formatters. that is, - string.__format__, etc. */ - -#include <locale.h> - -/* Before including this, you must include either: - stringlib/unicodedefs.h - stringlib/stringdefs.h - - Also, you should define the names: - FORMAT_STRING - FORMAT_LONG - FORMAT_FLOAT - FORMAT_COMPLEX - to be whatever you want the public names of these functions to - be. These are the only non-static functions defined here. -*/ - -/* Raises an exception about an unknown presentation type for this - * type. */ - -static void -unknown_presentation_type(STRINGLIB_CHAR presentation_type, - const char* type_name) -{ -#if STRINGLIB_IS_UNICODE - /* If STRINGLIB_CHAR is Py_UNICODE, %c might be out-of-range, - hence the two cases. If it is char, gcc complains that the - condition below is always true, hence the ifdef. */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Unknown format code '%c' " - "for object of type '%.200s'", - (char)presentation_type, - type_name); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Unknown format code '\\x%x' " - "for object of type '%.200s'", - (unsigned int)presentation_type, - type_name); -#endif -} - -static void -invalid_comma_type(STRINGLIB_CHAR presentation_type) -{ -#if STRINGLIB_IS_UNICODE - /* See comment in unknown_presentation_type */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '%c'.", - (char)presentation_type); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '\\x%x'.", - (unsigned int)presentation_type); -#endif -} - -/* - get_integer consumes 0 or more decimal digit characters from an - input string, updates *result with the corresponding positive - integer, and returns the number of digits consumed. - - returns -1 on error. -*/ -static int -get_integer(STRINGLIB_CHAR **ptr, STRINGLIB_CHAR *end, - Py_ssize_t *result) -{ - Py_ssize_t accumulator, digitval, oldaccumulator; - int numdigits; - accumulator = numdigits = 0; - for (;;(*ptr)++, numdigits++) { - if (*ptr >= end) - break; - digitval = STRINGLIB_TODECIMAL(**ptr); - if (digitval < 0) - break; - /* - This trick was copied from old Unicode format code. It's cute, - but would really suck on an old machine with a slow divide - implementation. Fortunately, in the normal case we do not - expect too many digits. - */ - oldaccumulator = accumulator; - accumulator *= 10; - if ((accumulator+10)/10 != oldaccumulator+1) { - PyErr_Format(PyExc_ValueError, - "Too many decimal digits in format string"); - return -1; - } - accumulator += digitval; - } - *result = accumulator; - return numdigits; -} - -/************************************************************************/ -/*********** standard format specifier parsing **************************/ -/************************************************************************/ - -/* returns true if this character is a specifier alignment token */ -Py_LOCAL_INLINE(int) -is_alignment_token(STRINGLIB_CHAR c) -{ - switch (c) { - case '<': case '>': case '=': case '^': - return 1; - default: - return 0; - } -} - -/* returns true if this character is a sign element */ -Py_LOCAL_INLINE(int) -is_sign_element(STRINGLIB_CHAR c) -{ - switch (c) { - case ' ': case '+': case '-': - return 1; - default: - return 0; - } -} - - -typedef struct { - STRINGLIB_CHAR fill_char; - STRINGLIB_CHAR align; - int alternate; - STRINGLIB_CHAR sign; - Py_ssize_t width; - int thousands_separators; - Py_ssize_t precision; - STRINGLIB_CHAR type; -} InternalFormatSpec; - - -#if 0 -/* Occassionally useful for debugging. Should normally be commented out. */ -static void -DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format) -{ - printf("internal format spec: fill_char %d\n", format->fill_char); - printf("internal format spec: align %d\n", format->align); - printf("internal format spec: alternate %d\n", format->alternate); - printf("internal format spec: sign %d\n", format->sign); - printf("internal format spec: width %zd\n", format->width); - printf("internal format spec: thousands_separators %d\n", - format->thousands_separators); - printf("internal format spec: precision %zd\n", format->precision); - printf("internal format spec: type %c\n", format->type); - printf("\n"); -} -#endif - - -/* - ptr points to the start of the format_spec, end points just past its end. - fills in format with the parsed information. - returns 1 on success, 0 on failure. - if failure, sets the exception -*/ -static int -parse_internal_render_format_spec(STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - InternalFormatSpec *format, - char default_type, - char default_align) -{ - STRINGLIB_CHAR *ptr = format_spec; - STRINGLIB_CHAR *end = format_spec + format_spec_len; - - /* end-ptr is used throughout this code to specify the length of - the input string */ - - Py_ssize_t consumed; - int align_specified = 0; - - format->fill_char = '\0'; - format->align = default_align; - format->alternate = 0; - format->sign = '\0'; - format->width = -1; - format->thousands_separators = 0; - format->precision = -1; - format->type = default_type; - - /* If the second char is an alignment token, - then parse the fill char */ - if (end-ptr >= 2 && is_alignment_token(ptr[1])) { - format->align = ptr[1]; - format->fill_char = ptr[0]; - align_specified = 1; - ptr += 2; - } - else if (end-ptr >= 1 && is_alignment_token(ptr[0])) { - format->align = ptr[0]; - align_specified = 1; - ++ptr; - } - - /* Parse the various sign options */ - if (end-ptr >= 1 && is_sign_element(ptr[0])) { - format->sign = ptr[0]; - ++ptr; - } - - /* If the next character is #, we're in alternate mode. This only - applies to integers. */ - if (end-ptr >= 1 && ptr[0] == '#') { - format->alternate = 1; - ++ptr; - } - - /* The special case for 0-padding (backwards compat) */ - if (format->fill_char == '\0' && end-ptr >= 1 && ptr[0] == '0') { - format->fill_char = '0'; - if (!align_specified) { - format->align = '='; - } - ++ptr; - } - - consumed = get_integer(&ptr, end, &format->width); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* If consumed is 0, we didn't consume any characters for the - width. In that case, reset the width to -1, because - get_integer() will have set it to zero. -1 is how we record - that the width wasn't specified. */ - if (consumed == 0) - format->width = -1; - - /* Comma signifies add thousands separators */ - if (end-ptr && ptr[0] == ',') { - format->thousands_separators = 1; - ++ptr; - } - - /* Parse field precision */ - if (end-ptr && ptr[0] == '.') { - ++ptr; - - consumed = get_integer(&ptr, end, &format->precision); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* Not having a precision after a dot is an error. */ - if (consumed == 0) { - PyErr_Format(PyExc_ValueError, - "Format specifier missing precision"); - return 0; - } - - } - - /* Finally, parse the type field. */ - - if (end-ptr > 1) { - /* More than one char remain, invalid conversion spec. */ - PyErr_Format(PyExc_ValueError, "Invalid conversion specification"); - return 0; - } - - if (end-ptr == 1) { - format->type = ptr[0]; - ++ptr; - } - - /* Do as much validating as we can, just by looking at the format - specifier. Do not take into account what type of formatting - we're doing (int, float, string). */ - - if (format->thousands_separators) { - switch (format->type) { - case 'd': - case 'e': - case 'f': - case 'g': - case 'E': - case 'G': - case '%': - case 'F': - case '\0': - /* These are allowed. See PEP 378.*/ - break; - default: - invalid_comma_type(format->type); - return 0; - } - } - - return 1; -} - -/* Calculate the padding needed. */ -static void -calc_padding(Py_ssize_t nchars, Py_ssize_t width, STRINGLIB_CHAR align, - Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding, - Py_ssize_t *n_total) -{ - if (width >= 0) { - if (nchars > width) - *n_total = nchars; - else - *n_total = width; - } - else { - /* not specified, use all of the chars and no more */ - *n_total = nchars; - } - - /* Figure out how much leading space we need, based on the - aligning */ - if (align == '>') - *n_lpadding = *n_total - nchars; - else if (align == '^') - *n_lpadding = (*n_total - nchars) / 2; - else if (align == '<' || align == '=') - *n_lpadding = 0; - else { - /* We should never have an unspecified alignment. */ - *n_lpadding = 0; - assert(0); - } - - *n_rpadding = *n_total - nchars - *n_lpadding; -} - -/* Do the padding, and return a pointer to where the caller-supplied - content goes. */ -static STRINGLIB_CHAR * -fill_padding(STRINGLIB_CHAR *p, Py_ssize_t nchars, STRINGLIB_CHAR fill_char, - Py_ssize_t n_lpadding, Py_ssize_t n_rpadding) -{ - /* Pad on left. */ - if (n_lpadding) - STRINGLIB_FILL(p, fill_char, n_lpadding); - - /* Pad on right. */ - if (n_rpadding) - STRINGLIB_FILL(p + nchars + n_lpadding, fill_char, n_rpadding); - - /* Pointer to the user content. */ - return p + n_lpadding; -} - -#if defined FORMAT_FLOAT || defined FORMAT_LONG || defined FORMAT_COMPLEX -/************************************************************************/ -/*********** common routines for numeric formatting *********************/ -/************************************************************************/ - -/* Locale type codes. */ -#define LT_CURRENT_LOCALE 0 -#define LT_DEFAULT_LOCALE 1 -#define LT_NO_LOCALE 2 - -/* Locale info needed for formatting integers and the part of floats - before and including the decimal. Note that locales only support - 8-bit chars, not unicode. */ -typedef struct { - char *decimal_point; - char *thousands_sep; - char *grouping; -} LocaleInfo; - -/* describes the layout for an integer, see the comment in - calc_number_widths() for details */ -typedef struct { - Py_ssize_t n_lpadding; - Py_ssize_t n_prefix; - Py_ssize_t n_spadding; - Py_ssize_t n_rpadding; - char sign; - Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */ - Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including - any grouping chars. */ - Py_ssize_t n_decimal; /* 0 if only an integer */ - Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part, - excluding the decimal itself, if - present. */ - - /* These 2 are not the widths of fields, but are needed by - STRINGLIB_GROUPING. */ - Py_ssize_t n_digits; /* The number of digits before a decimal - or exponent. */ - Py_ssize_t n_min_width; /* The min_width we used when we computed - the n_grouped_digits width. */ -} NumberFieldWidths; - - -/* Given a number of the form: - digits[remainder] - where ptr points to the start and end points to the end, find where - the integer part ends. This could be a decimal, an exponent, both, - or neither. - If a decimal point is present, set *has_decimal and increment - remainder beyond it. - Results are undefined (but shouldn't crash) for improperly - formatted strings. -*/ -static void -parse_number(STRINGLIB_CHAR *ptr, Py_ssize_t len, - Py_ssize_t *n_remainder, int *has_decimal) -{ - STRINGLIB_CHAR *end = ptr + len; - STRINGLIB_CHAR *remainder; - - while (ptr<end && isdigit(*ptr)) - ++ptr; - remainder = ptr; - - /* Does remainder start with a decimal point? */ - *has_decimal = ptr<end && *remainder == '.'; - - /* Skip the decimal point. */ - if (*has_decimal) - remainder++; - - *n_remainder = end - remainder; -} - -/* not all fields of format are used. for example, precision is - unused. should this take discrete params in order to be more clear - about what it does? or is passing a single format parameter easier - and more efficient enough to justify a little obfuscation? */ -static Py_ssize_t -calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, - STRINGLIB_CHAR sign_char, STRINGLIB_CHAR *number, - Py_ssize_t n_number, Py_ssize_t n_remainder, - int has_decimal, const LocaleInfo *locale, - const InternalFormatSpec *format) -{ - Py_ssize_t n_non_digit_non_padding; - Py_ssize_t n_padding; - - spec->n_digits = n_number - n_remainder - (has_decimal?1:0); - spec->n_lpadding = 0; - spec->n_prefix = n_prefix; - spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0; - spec->n_remainder = n_remainder; - spec->n_spadding = 0; - spec->n_rpadding = 0; - spec->sign = '\0'; - spec->n_sign = 0; - - /* the output will look like: - | | - | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> | - | | - - sign is computed from format->sign and the actual - sign of the number - - prefix is given (it's for the '0x' prefix) - - digits is already known - - the total width is either given, or computed from the - actual digits - - only one of lpadding, spadding, and rpadding can be non-zero, - and it's calculated from the width and other fields - */ - - /* compute the various parts we're going to write */ - switch (format->sign) { - case '+': - /* always put a + or - */ - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : '+'); - break; - case ' ': - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : ' '); - break; - default: - /* Not specified, or the default (-) */ - if (sign_char == '-') { - spec->n_sign = 1; - spec->sign = '-'; - } - } - - /* The number of chars used for non-digits and non-padding. */ - n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal + - spec->n_remainder; - - /* min_width can go negative, that's okay. format->width == -1 means - we don't care. */ - if (format->fill_char == '0' && format->align == '=') - spec->n_min_width = format->width - n_non_digit_non_padding; - else - spec->n_min_width = 0; - - if (spec->n_digits == 0) - /* This case only occurs when using 'c' formatting, we need - to special case it because the grouping code always wants - to have at least one character. */ - spec->n_grouped_digits = 0; - else - spec->n_grouped_digits = STRINGLIB_GROUPING(NULL, 0, NULL, - spec->n_digits, - spec->n_min_width, - locale->grouping, - locale->thousands_sep); - - /* Given the desired width and the total of digit and non-digit - space we consume, see if we need any padding. format->width can - be negative (meaning no padding), but this code still works in - that case. */ - n_padding = format->width - - (n_non_digit_non_padding + spec->n_grouped_digits); - if (n_padding > 0) { - /* Some padding is needed. Determine if it's left, space, or right. */ - switch (format->align) { - case '<': - spec->n_rpadding = n_padding; - break; - case '^': - spec->n_lpadding = n_padding / 2; - spec->n_rpadding = n_padding - spec->n_lpadding; - break; - case '=': - spec->n_spadding = n_padding; - break; - case '>': - spec->n_lpadding = n_padding; - break; - default: - /* Shouldn't get here, but treat it as '>' */ - spec->n_lpadding = n_padding; - assert(0); - break; - } - } - return spec->n_lpadding + spec->n_sign + spec->n_prefix + - spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + - spec->n_remainder + spec->n_rpadding; -} - -/* Fill in the digit parts of a numbers's string representation, - as determined in calc_number_widths(). - No error checking, since we know the buffer is the correct size. */ -static void -fill_number(STRINGLIB_CHAR *buf, const NumberFieldWidths *spec, - STRINGLIB_CHAR *digits, Py_ssize_t n_digits, - STRINGLIB_CHAR *prefix, STRINGLIB_CHAR fill_char, - LocaleInfo *locale, int toupper) -{ - /* Used to keep track of digits, decimal, and remainder. */ - STRINGLIB_CHAR *p = digits; - -#ifndef NDEBUG - Py_ssize_t r; -#endif - - if (spec->n_lpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_lpadding); - buf += spec->n_lpadding; - } - if (spec->n_sign == 1) { - *buf++ = spec->sign; - } - if (spec->n_prefix) { - memmove(buf, - prefix, - spec->n_prefix * sizeof(STRINGLIB_CHAR)); - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_prefix; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_prefix; - } - if (spec->n_spadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_spadding); - buf += spec->n_spadding; - } - - /* Only for type 'c' special case, it has no digits. */ - if (spec->n_digits != 0) { - /* Fill the digits with InsertThousandsGrouping. */ -#ifndef NDEBUG - r = -#endif - STRINGLIB_GROUPING(buf, spec->n_grouped_digits, digits, - spec->n_digits, spec->n_min_width, - locale->grouping, locale->thousands_sep); -#ifndef NDEBUG - assert(r == spec->n_grouped_digits); -#endif - p += spec->n_digits; - } - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_grouped_digits; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_grouped_digits; - - if (spec->n_decimal) { - Py_ssize_t t; - for (t = 0; t < spec->n_decimal; ++t) - buf[t] = locale->decimal_point[t]; - buf += spec->n_decimal; - p += 1; - } - - if (spec->n_remainder) { - memcpy(buf, p, spec->n_remainder * sizeof(STRINGLIB_CHAR)); - buf += spec->n_remainder; - p += spec->n_remainder; - } - - if (spec->n_rpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_rpadding); - buf += spec->n_rpadding; - } -} - -static char no_grouping[1] = {CHAR_MAX}; - -/* Find the decimal point character(s?), thousands_separator(s?), and - grouping description, either for the current locale if type is - LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or - none if LT_NO_LOCALE. */ -static void -get_locale_info(int type, LocaleInfo *locale_info) -{ - switch (type) { - case LT_CURRENT_LOCALE: { - struct lconv *locale_data = localeconv(); - locale_info->decimal_point = locale_data->decimal_point; - locale_info->thousands_sep = locale_data->thousands_sep; - locale_info->grouping = locale_data->grouping; - break; - } - case LT_DEFAULT_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ","; - locale_info->grouping = "\3"; /* Group every 3 characters. The - (implicit) trailing 0 means repeat - infinitely. */ - break; - case LT_NO_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ""; - locale_info->grouping = no_grouping; - break; - default: - assert(0); - } -} - -#endif /* FORMAT_FLOAT || FORMAT_LONG || FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** string formatting ******************************************/ -/************************************************************************/ - -static PyObject * -format_string_internal(PyObject *value, const InternalFormatSpec *format) -{ - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - STRINGLIB_CHAR *p; - Py_ssize_t len = STRINGLIB_LEN(value); - PyObject *result = NULL; - - /* sign is not allowed on strings */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed in string format specifier"); - goto done; - } - - /* alternate is not allowed on strings */ - if (format->alternate) { - PyErr_SetString(PyExc_ValueError, - "Alternate form (#) not allowed in string format " - "specifier"); - goto done; - } - - /* '=' alignment not allowed on strings */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment not allowed " - "in string format specifier"); - goto done; - } - - /* if precision is specified, output no more that format.precision - characters */ - if (format->precision >= 0 && len >= format->precision) { - len = format->precision; - } - - calc_padding(len, format->width, format->align, &lpad, &rpad, &total); - - /* allocate the resulting string */ - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Write into that space. First the padding. */ - p = fill_padding(STRINGLIB_STR(result), len, - format->fill_char=='\0'?' ':format->fill_char, - lpad, rpad); - - /* Then the source string. */ - memcpy(p, STRINGLIB_STR(value), len * sizeof(STRINGLIB_CHAR)); - -done: - return result; -} - - -/************************************************************************/ -/*********** long formatting ********************************************/ -/************************************************************************/ - -#if defined FORMAT_LONG || defined FORMAT_INT -typedef PyObject* -(*IntOrLongToString)(PyObject *value, int base); - -static PyObject * -format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - STRINGLIB_CHAR *pnumeric_chars; - STRINGLIB_CHAR numeric_char; - STRINGLIB_CHAR sign_char = '\0'; - Py_ssize_t n_digits; /* count of digits need from the computed - string */ - Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which - produces non-digits */ - Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */ - Py_ssize_t n_total; - STRINGLIB_CHAR *prefix = NULL; - NumberFieldWidths spec; - long x; - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* no precision allowed on integers */ - if (format->precision != -1) { - PyErr_SetString(PyExc_ValueError, - "Precision not allowed in integer format specifier"); - goto done; - } - - /* special case for character formatting */ - if (format->type == 'c') { - /* error to specify a sign */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed with integer" - " format specifier 'c'"); - goto done; - } - - /* taken from unicodeobject.c formatchar() */ - /* Integer input truncated to a character */ -/* XXX: won't work for int */ - x = PyLong_AsLong(value); - if (x == -1 && PyErr_Occurred()) - goto done; -#ifdef Py_UNICODE_WIDE - if (x < 0 || x > 0x10ffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000) " - "(wide Python build)"); - goto done; - } -#else - if (x < 0 || x > 0xffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x10000) " - "(narrow Python build)"); - goto done; - } -#endif - numeric_char = (STRINGLIB_CHAR)x; - pnumeric_chars = &numeric_char; - n_digits = 1; - - /* As a sort-of hack, we tell calc_number_widths that we only - have "remainder" characters. calc_number_widths thinks - these are characters that don't get formatted, only copied - into the output string. We do this for 'c' formatting, - because the characters are likely to be non-digits. */ - n_remainder = 1; - } - else { - int base; - int leading_chars_to_skip = 0; /* Number of characters added by - PyNumber_ToBase that we want to - skip over. */ - - /* Compute the base and how many characters will be added by - PyNumber_ToBase */ - switch (format->type) { - case 'b': - base = 2; - leading_chars_to_skip = 2; /* 0b */ - break; - case 'o': - base = 8; - leading_chars_to_skip = 2; /* 0o */ - break; - case 'x': - case 'X': - base = 16; - leading_chars_to_skip = 2; /* 0x */ - break; - default: /* shouldn't be needed, but stops a compiler warning */ - case 'd': - case 'n': - base = 10; - break; - } - - /* The number of prefix chars is the same as the leading - chars to skip */ - if (format->alternate) - n_prefix = leading_chars_to_skip; - - /* Do the hard part, converting to a string in a given base */ - tmp = tostring(value, base); - if (tmp == NULL) - goto done; - - pnumeric_chars = STRINGLIB_STR(tmp); - n_digits = STRINGLIB_LEN(tmp); - - prefix = pnumeric_chars; - - /* Remember not to modify what pnumeric_chars points to. it - might be interned. Only modify it after we copy it into a - newly allocated output buffer. */ - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (pnumeric_chars[0] == '-') { - sign_char = pnumeric_chars[0]; - ++prefix; - ++leading_chars_to_skip; - } - - /* Skip over the leading chars (0x, 0b, etc.) */ - n_digits -= leading_chars_to_skip; - pnumeric_chars += leading_chars_to_skip; - } - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, n_prefix, sign_char, pnumeric_chars, - n_digits, n_remainder, 0, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (!result) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, pnumeric_chars, n_digits, - prefix, format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* defined FORMAT_LONG || defined FORMAT_INT */ - -/************************************************************************/ -/*********** float formatting *******************************************/ -/************************************************************************/ - -#ifdef FORMAT_FLOAT -#if STRINGLIB_IS_UNICODE -static void -strtounicode(Py_UNICODE *buffer, const char *charbuffer, Py_ssize_t len) -{ - Py_ssize_t i; - for (i = 0; i < len; ++i) - buffer[i] = (Py_UNICODE)charbuffer[i]; -} -#endif - -/* much of this is taken from unicodeobject.c */ -static PyObject * -format_float_internal(PyObject *value, - const InternalFormatSpec *format) -{ - char *buf = NULL; /* buffer returned from PyOS_double_to_string */ - Py_ssize_t n_digits; - Py_ssize_t n_remainder; - Py_ssize_t n_total; - int has_decimal; - double val; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - int add_pct = 0; - STRINGLIB_CHAR *p; - NumberFieldWidths spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR sign_char = '\0'; - int float_type; /* Used to see if we have a nan, inf, or regular float. */ - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Behaves in the same way as repr(x) - and str(x) if no precision is given, else like 'g', but with - at least one digit after the decimal point. */ - flags |= Py_DTSF_ADD_DOT_0; - type = 'r'; - default_precision = 0; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - val = PyFloat_AsDouble(value); - if (val == -1.0 && PyErr_Occurred()) - goto done; - - if (type == '%') { - type = 'f'; - val *= 100; - add_pct = 1; - } - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - buf = PyOS_double_to_string(val, (char)type, precision, flags, - &float_type); - if (buf == NULL) - goto done; - n_digits = strlen(buf); - - if (add_pct) { - /* We know that buf has a trailing zero (since we just called - strlen() on it), and we don't use that fact any more. So we - can just write over the trailing zero. */ - buf[n_digits] = '%'; - n_digits += 1; - } - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_digits)*sizeof(Py_UNICODE)); - if (unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(unicode_tmp, buf, n_digits); - p = unicode_tmp; -#else - p = buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p == '-') { - sign_char = *p; - ++p; - --n_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p, n_digits, &n_remainder, &has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, 0, sign_char, p, n_digits, - n_remainder, has_decimal, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (result == NULL) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, p, n_digits, NULL, - format->fill_char == '\0' ? ' ' : format->fill_char, &locale, - 0); - -done: - PyMem_Free(buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_FLOAT */ - -/************************************************************************/ -/*********** complex formatting *****************************************/ -/************************************************************************/ - -#ifdef FORMAT_COMPLEX - -static PyObject * -format_complex_internal(PyObject *value, - const InternalFormatSpec *format) -{ - double re; - double im; - char *re_buf = NULL; /* buffer returned from PyOS_double_to_string */ - char *im_buf = NULL; /* buffer returned from PyOS_double_to_string */ - - InternalFormatSpec tmp_format = *format; - Py_ssize_t n_re_digits; - Py_ssize_t n_im_digits; - Py_ssize_t n_re_remainder; - Py_ssize_t n_im_remainder; - Py_ssize_t n_re_total; - Py_ssize_t n_im_total; - int re_has_decimal; - int im_has_decimal; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - STRINGLIB_CHAR *p_re; - STRINGLIB_CHAR *p_im; - NumberFieldWidths re_spec; - NumberFieldWidths im_spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR *p; - STRINGLIB_CHAR re_sign_char = '\0'; - STRINGLIB_CHAR im_sign_char = '\0'; - int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ - int im_float_type; - int add_parens = 0; - int skip_re = 0; - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *re_unicode_tmp = NULL; - Py_UNICODE *im_unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* Zero padding is not allowed. */ - if (format->fill_char == '0') { - PyErr_SetString(PyExc_ValueError, - "Zero padding is not allowed in complex format " - "specifier"); - goto done; - } - - /* Neither is '=' alignment . */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment flag is not allowed in complex format " - "specifier"); - goto done; - } - - re = PyComplex_RealAsDouble(value); - if (re == -1.0 && PyErr_Occurred()) - goto done; - im = PyComplex_ImagAsDouble(value); - if (im == -1.0 && PyErr_Occurred()) - goto done; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Should be like str(self). */ - type = 'r'; - default_precision = 0; - if (re == 0.0 && copysign(1.0, re) == 1.0) - skip_re = 1; - else - add_parens = 1; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - re_buf = PyOS_double_to_string(re, (char)type, precision, flags, - &re_float_type); - if (re_buf == NULL) - goto done; - im_buf = PyOS_double_to_string(im, (char)type, precision, flags, - &im_float_type); - if (im_buf == NULL) - goto done; - - n_re_digits = strlen(re_buf); - n_im_digits = strlen(im_buf); - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - re_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_re_digits)*sizeof(Py_UNICODE)); - if (re_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(re_unicode_tmp, re_buf, n_re_digits); - p_re = re_unicode_tmp; - - im_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_im_digits)*sizeof(Py_UNICODE)); - if (im_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(im_unicode_tmp, im_buf, n_im_digits); - p_im = im_unicode_tmp; -#else - p_re = re_buf; - p_im = im_buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p_re == '-') { - re_sign_char = *p_re; - ++p_re; - --n_re_digits; - } - if (*p_im == '-') { - im_sign_char = *p_im; - ++p_im; - --n_im_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p_re, n_re_digits, &n_re_remainder, &re_has_decimal); - parse_number(p_im, n_im_digits, &n_im_remainder, &im_has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Turn off any padding. We'll do it later after we've composed - the numbers without padding. */ - tmp_format.fill_char = '\0'; - tmp_format.align = '<'; - tmp_format.width = -1; - - /* Calculate how much memory we'll need. */ - n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, p_re, - n_re_digits, n_re_remainder, - re_has_decimal, &locale, &tmp_format); - - /* Same formatting, but always include a sign, unless the real part is - * going to be omitted, in which case we use whatever sign convention was - * requested by the original format. */ - if (!skip_re) - tmp_format.sign = '+'; - n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, p_im, - n_im_digits, n_im_remainder, - im_has_decimal, &locale, &tmp_format); - - if (skip_re) - n_re_total = 0; - - /* Add 1 for the 'j', and optionally 2 for parens. */ - calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, - format->width, format->align, &lpad, &rpad, &total); - - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Populate the memory. First, the padding. */ - p = fill_padding(STRINGLIB_STR(result), - n_re_total + n_im_total + 1 + add_parens * 2, - format->fill_char=='\0' ? ' ' : format->fill_char, - lpad, rpad); - - if (add_parens) - *p++ = '('; - - if (!skip_re) { - fill_number(p, &re_spec, p_re, n_re_digits, NULL, 0, &locale, 0); - p += n_re_total; - } - fill_number(p, &im_spec, p_im, n_im_digits, NULL, 0, &locale, 0); - p += n_im_total; - *p++ = 'j'; - - if (add_parens) - *p++ = ')'; - -done: - PyMem_Free(re_buf); - PyMem_Free(im_buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(re_unicode_tmp); - PyMem_Free(im_unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** built in formatters ****************************************/ -/************************************************************************/ -PyObject * -FORMAT_STRING(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - InternalFormatSpec format; - PyObject *result = NULL; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, format_spec_len, - &format, 's', '<')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 's': - /* no type conversion needed, already a string. do the formatting */ - result = format_string_internal(obj, &format); - break; - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} - -#if defined FORMAT_LONG || defined FORMAT_INT -static PyObject* -format_int_or_long(PyObject* obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, 'd', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 'b': - case 'c': - case 'd': - case 'o': - case 'x': - case 'X': - case 'n': - /* no type conversion needed, already an int (or long). do - the formatting */ - result = format_int_or_long_internal(obj, &format, tostring); - break; - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case '%': - /* convert to float */ - tmp = PyNumber_Float(obj); - if (tmp == NULL) - goto done; - result = format_float_internal(tmp, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* FORMAT_LONG || defined FORMAT_INT */ - -#ifdef FORMAT_LONG -/* Need to define long_format as a function that will convert a long - to a string. In 3.0, _PyLong_Format has the correct signature. In - 2.x, we need to fudge a few parameters */ -#if PY_VERSION_HEX >= 0x03000000 -#define long_format _PyLong_Format -#else -static PyObject* -long_format(PyObject* value, int base) -{ - /* Convert to base, don't add trailing 'L', and use the new octal - format. We already know this is a long object */ - assert(PyLong_Check(value)); - /* convert to base, don't add 'L', and use the new octal format */ - return _PyLong_Format(value, base, 0, 1); -} -#endif - -PyObject * -FORMAT_LONG(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - long_format); -} -#endif /* FORMAT_LONG */ - -#ifdef FORMAT_INT -/* this is only used for 2.x, not 3.0 */ -static PyObject* -int_format(PyObject* value, int base) -{ - /* Convert to base, and use the new octal format. We already - know this is an int object */ - assert(PyInt_Check(value)); - return _PyInt_Format((PyIntObject*)value, base, 1); -} - -PyObject * -FORMAT_INT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - int_format); -} -#endif /* FORMAT_INT */ - -#ifdef FORMAT_FLOAT -PyObject * -FORMAT_FLOAT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - case '%': - /* no conversion, already a float. do the formatting */ - result = format_float_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_FLOAT */ - -#ifdef FORMAT_COMPLEX -PyObject * -FORMAT_COMPLEX(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - /* no conversion, already a complex. do the formatting */ - result = format_complex_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_COMPLEX */ diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h index f548133875..6e2f07342c 100644 --- a/Objects/stringlib/localeutil.h +++ b/Objects/stringlib/localeutil.h @@ -1,21 +1,19 @@ /* stringlib: locale related helpers implementation */ -#ifndef STRINGLIB_LOCALEUTIL_H -#define STRINGLIB_LOCALEUTIL_H - #include <locale.h> -#define MAX(x, y) ((x) < (y) ? (y) : (x)) -#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#ifndef STRINGLIB_IS_UNICODE +# error "localeutil is specific to Unicode" +#endif typedef struct { const char *grouping; char previous; Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; +} STRINGLIB(GroupGenerator); static void -_GroupGenerator_init(GroupGenerator *self, const char *grouping) +STRINGLIB(GroupGenerator_init)(STRINGLIB(GroupGenerator) *self, const char *grouping) { self->grouping = grouping; self->i = 0; @@ -24,7 +22,7 @@ _GroupGenerator_init(GroupGenerator *self, const char *grouping) /* Returns the next grouping, or 0 to signify end. */ static Py_ssize_t -_GroupGenerator_next(GroupGenerator *self) +STRINGLIB(GroupGenerator_next)(STRINGLIB(GroupGenerator) *self) { /* Note that we don't really do much error checking here. If a grouping string contains just CHAR_MAX, for example, then just @@ -48,27 +46,18 @@ _GroupGenerator_next(GroupGenerator *self) /* Fill in some digits, leading zeros, and thousands separator. All are optional, depending on when we're called. */ static void -fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, - Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep, +STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, + Py_ssize_t n_chars, Py_ssize_t n_zeros, STRINGLIB_CHAR* thousands_sep, Py_ssize_t thousands_sep_len) { -#if STRINGLIB_IS_UNICODE Py_ssize_t i; -#endif if (thousands_sep) { *buffer_end -= thousands_sep_len; /* Copy the thousands_sep chars into the buffer. */ -#if STRINGLIB_IS_UNICODE - /* Convert from the char's of the thousands_sep from - the locale into unicode. */ - for (i = 0; i < thousands_sep_len; ++i) - (*buffer_end)[i] = thousands_sep[i]; -#else - /* No conversion, just memcpy the thousands_sep. */ - memcpy(*buffer_end, thousands_sep, thousands_sep_len); -#endif + memcpy(*buffer_end, thousands_sep, + thousands_sep_len * STRINGLIB_SIZEOF_CHAR); } *buffer_end -= n_chars; @@ -76,11 +65,12 @@ fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, memcpy(*buffer_end, *digits_end, n_chars * sizeof(STRINGLIB_CHAR)); *buffer_end -= n_zeros; - STRINGLIB_FILL(*buffer_end, '0', n_zeros); + for (i = 0; i < n_zeros; i++) + (*buffer_end)[i] = '0'; } /** - * _Py_InsertThousandsGrouping: + * InsertThousandsGrouping: * @buffer: A pointer to the start of a string. * @n_buffer: Number of characters in @buffer. * @digits: A pointer to the digits we're reading from. If count @@ -109,14 +99,16 @@ fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, * As closely as possible, this code mimics the logic in decimal.py's _insert_thousands_sep(). **/ -Py_ssize_t -_Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, - Py_ssize_t n_buffer, - STRINGLIB_CHAR *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - const char *thousands_sep) +static Py_ssize_t +STRINGLIB(InsertThousandsGrouping)( + STRINGLIB_CHAR *buffer, + Py_ssize_t n_buffer, + STRINGLIB_CHAR *digits, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + STRINGLIB_CHAR *thousands_sep, + Py_ssize_t thousands_sep_len) { Py_ssize_t count = 0; Py_ssize_t n_zeros; @@ -128,23 +120,22 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, STRINGLIB_CHAR *digits_end = NULL; Py_ssize_t l; Py_ssize_t n_chars; - Py_ssize_t thousands_sep_len = strlen(thousands_sep); Py_ssize_t remaining = n_digits; /* Number of chars remaining to be looked at */ /* A generator that returns all of the grouping widths, until it returns 0. */ - GroupGenerator groupgen; - _GroupGenerator_init(&groupgen, grouping); + STRINGLIB(GroupGenerator) groupgen; + STRINGLIB(GroupGenerator_init)(&groupgen, grouping); if (buffer) { buffer_end = buffer + n_buffer; digits_end = digits + n_digits; } - while ((l = _GroupGenerator_next(&groupgen)) > 0) { - l = MIN(l, MAX(MAX(remaining, min_width), 1)); - n_zeros = MAX(0, l - remaining); - n_chars = MAX(0, MIN(remaining, l)); + while ((l = STRINGLIB(GroupGenerator_next)(&groupgen)) > 0) { + l = Py_MIN(l, Py_MAX(Py_MAX(remaining, min_width), 1)); + n_zeros = Py_MAX(0, l - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, l)); /* Use n_zero zero's and n_chars chars */ @@ -153,7 +144,7 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } @@ -172,41 +163,18 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, if (!loop_broken) { /* We left the loop without using a break statement. */ - l = MAX(MAX(remaining, min_width), 1); - n_zeros = MAX(0, l - remaining); - n_chars = MAX(0, MIN(remaining, l)); + l = Py_MAX(Py_MAX(remaining, min_width), 1); + n_zeros = Py_MAX(0, l - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, l)); /* Use n_zero zero's and n_chars chars */ count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } } return count; } -/** - * _Py_InsertThousandsGroupingLocale: - * @buffer: A pointer to the start of a string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * - * Reads thee current locale and calls _Py_InsertThousandsGrouping(). - **/ -Py_ssize_t -_Py_InsertThousandsGroupingLocale(STRINGLIB_CHAR *buffer, - Py_ssize_t n_buffer, - STRINGLIB_CHAR *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width) -{ - struct lconv *locale_data = localeconv(); - const char *grouping = locale_data->grouping; - const char *thousands_sep = locale_data->thousands_sep; - - return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits, - min_width, grouping, thousands_sep); -} -#endif /* STRINGLIB_LOCALEUTIL_H */ diff --git a/Objects/stringlib/partition.h b/Objects/stringlib/partition.h index 0170bddbf0..40cb5129d5 100644 --- a/Objects/stringlib/partition.h +++ b/Objects/stringlib/partition.h @@ -1,14 +1,11 @@ /* stringlib: partition implementation */ -#ifndef STRINGLIB_PARTITION_H -#define STRINGLIB_PARTITION_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(PyObject*) -stringlib_partition(PyObject* str_obj, +STRINGLIB(partition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -25,7 +22,7 @@ stringlib_partition(PyObject* str_obj, if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -58,7 +55,7 @@ stringlib_partition(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject*) -stringlib_rpartition(PyObject* str_obj, +STRINGLIB(rpartition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -75,7 +72,7 @@ stringlib_rpartition(PyObject* str_obj, if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -107,4 +104,3 @@ stringlib_rpartition(PyObject* str_obj, return out; } -#endif diff --git a/Objects/stringlib/split.h b/Objects/stringlib/split.h index 60e77674f0..947dd28e6c 100644 --- a/Objects/stringlib/split.h +++ b/Objects/stringlib/split.h @@ -1,8 +1,5 @@ /* stringlib: split implementation */ -#ifndef STRINGLIB_SPLIT_H -#define STRINGLIB_SPLIT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif @@ -54,7 +51,7 @@ #define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count Py_LOCAL_INLINE(PyObject *) -stringlib_split_whitespace(PyObject* str_obj, +STRINGLIB(split_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -102,7 +99,7 @@ stringlib_split_whitespace(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_split_char(PyObject* str_obj, +STRINGLIB(split_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -145,7 +142,7 @@ stringlib_split_char(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_split(PyObject* str_obj, +STRINGLIB(split)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -158,7 +155,7 @@ stringlib_split(PyObject* str_obj, return NULL; } else if (sep_len == 1) - return stringlib_split_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(split_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -166,7 +163,7 @@ stringlib_split(PyObject* str_obj, i = j = 0; while (maxcount-- > 0) { - pos = fastsearch(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) break; j = i + pos; @@ -193,7 +190,7 @@ stringlib_split(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_whitespace(PyObject* str_obj, +STRINGLIB(rsplit_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -243,7 +240,7 @@ stringlib_rsplit_whitespace(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_char(PyObject* str_obj, +STRINGLIB(rsplit_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -287,7 +284,7 @@ stringlib_rsplit_char(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit(PyObject* str_obj, +STRINGLIB(rsplit)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -300,7 +297,7 @@ stringlib_rsplit(PyObject* str_obj, return NULL; } else if (sep_len == 1) - return stringlib_rsplit_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(rsplit_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -308,7 +305,7 @@ stringlib_rsplit(PyObject* str_obj, j = str_len; while (maxcount-- > 0) { - pos = fastsearch(str, j, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, j, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) break; SPLIT_ADD(str, pos + sep_len, j); @@ -336,7 +333,7 @@ stringlib_rsplit(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_splitlines(PyObject* str_obj, +STRINGLIB(splitlines)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, int keepends) { @@ -391,4 +388,3 @@ stringlib_splitlines(PyObject* str_obj, return NULL; } -#endif diff --git a/Objects/stringlib/stringdefs.h b/Objects/stringlib/stringdefs.h index 1c49426ff6..7bb91a7a5b 100644 --- a/Objects/stringlib/stringdefs.h +++ b/Objects/stringlib/stringdefs.h @@ -6,7 +6,10 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 0 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyBytesObject +#define STRINGLIB_SIZEOF_CHAR 1 #define STRINGLIB_CHAR char #define STRINGLIB_TYPE_NAME "string" #define STRINGLIB_PARSE_CODE "S" @@ -15,9 +18,6 @@ #define STRINGLIB_ISLINEBREAK(x) ((x == '\n') || (x == '\r')) #define STRINGLIB_ISDECIMAL(x) ((x >= '0') && (x <= '9')) #define STRINGLIB_TODECIMAL(x) (STRINGLIB_ISDECIMAL(x) ? (x - '0') : -1) -#define STRINGLIB_TOUPPER Py_TOUPPER -#define STRINGLIB_TOLOWER Py_TOLOWER -#define STRINGLIB_FILL memset #define STRINGLIB_STR PyBytes_AS_STRING #define STRINGLIB_LEN PyBytes_GET_SIZE #define STRINGLIB_NEW PyBytes_FromStringAndSize @@ -25,7 +25,5 @@ #define STRINGLIB_CHECK PyBytes_Check #define STRINGLIB_CHECK_EXACT PyBytes_CheckExact #define STRINGLIB_TOSTR PyObject_Str -#define STRINGLIB_GROUPING _PyBytes_InsertThousandsGrouping -#define STRINGLIB_GROUPING_LOCALE _PyBytes_InsertThousandsGroupingLocale #define STRINGLIB_TOASCII PyObject_Repr #endif /* !STRINGLIB_STRINGDEFS_H */ diff --git a/Objects/stringlib/ucs1lib.h b/Objects/stringlib/ucs1lib.h new file mode 100644 index 0000000000..e8c6fcb85f --- /dev/null +++ b/Objects/stringlib/ucs1lib.h @@ -0,0 +1,31 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs1lib_fastsearch +#define STRINGLIB(F) ucs1lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 1 +#define STRINGLIB_MAX_CHAR 0xFFu +#define STRINGLIB_CHAR Py_UCS1 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS1 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs1_InsertThousandsGrouping + + diff --git a/Objects/stringlib/ucs2lib.h b/Objects/stringlib/ucs2lib.h new file mode 100644 index 0000000000..45e572963d --- /dev/null +++ b/Objects/stringlib/ucs2lib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs2lib_fastsearch +#define STRINGLIB(F) ucs2lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 2 +#define STRINGLIB_MAX_CHAR 0xFFFFu +#define STRINGLIB_CHAR Py_UCS2 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_2BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS2 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs2_InsertThousandsGrouping + diff --git a/Objects/stringlib/ucs4lib.h b/Objects/stringlib/ucs4lib.h new file mode 100644 index 0000000000..647a27e233 --- /dev/null +++ b/Objects/stringlib/ucs4lib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs4lib_fastsearch +#define STRINGLIB(F) ucs4lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 4 +#define STRINGLIB_MAX_CHAR 0x10FFFFu +#define STRINGLIB_CHAR Py_UCS4 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_4BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS4 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs4_InsertThousandsGrouping + diff --git a/Objects/stringlib/undef.h b/Objects/stringlib/undef.h new file mode 100644 index 0000000000..03117ec443 --- /dev/null +++ b/Objects/stringlib/undef.h @@ -0,0 +1,12 @@ +#undef FASTSEARCH +#undef STRINGLIB +#undef STRINGLIB_SIZEOF_CHAR +#undef STRINGLIB_MAX_CHAR +#undef STRINGLIB_CHAR +#undef STRINGLIB_STR +#undef STRINGLIB_LEN +#undef STRINGLIB_NEW +#undef STRINGLIB_RESIZE +#undef _Py_InsertThousandsGrouping +#undef STRINGLIB_IS_UNICODE + diff --git a/Objects/stringlib/string_format.h b/Objects/stringlib/unicode_format.h index 6c7adcb01e..be580c6858 100644 --- a/Objects/stringlib/string_format.h +++ b/Objects/stringlib/unicode_format.h @@ -1,16 +1,7 @@ /* - string_format.h -- implementation of string.format(). - - It uses the Objects/stringlib conventions, so that it can be - compiled for both unicode and string objects. + unicode_format.h -- implementation of str.format(). */ - -/* Defines for Python 2.6 compatibility */ -#if PY_VERSION_HEX < 0x03000000 -#define PyLong_FromSsize_t _PyLong_FromSsize_t -#endif - /* Defines for more efficiently reallocating the string buffer */ #define INITIAL_SIZE_INCREMENT 100 #define SIZE_MULTIPLIER 2 @@ -26,8 +17,8 @@ unicode pointers. */ typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; + PyObject *str; /* borrowed reference */ + Py_ssize_t start, end; } SubString; @@ -64,34 +55,32 @@ AutoNumber_Init(AutoNumber *auto_number) /* fill in a SubString from a pointer and length */ Py_LOCAL_INLINE(void) -SubString_init(SubString *str, STRINGLIB_CHAR *p, Py_ssize_t len) +SubString_init(SubString *str, PyObject *s, Py_ssize_t start, Py_ssize_t end) { - str->ptr = p; - if (p == NULL) - str->end = NULL; - else - str->end = str->ptr + len; + str->str = s; + str->start = start; + str->end = end; } -/* return a new string. if str->ptr is NULL, return None */ +/* return a new string. if str->str is NULL, return None */ Py_LOCAL_INLINE(PyObject *) SubString_new_object(SubString *str) { - if (str->ptr == NULL) { + if (str->str == NULL) { Py_INCREF(Py_None); return Py_None; } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); + return PyUnicode_Substring(str->str, str->start, str->end); } -/* return a new string. if str->ptr is NULL, return None */ +/* return a new string. if str->str is NULL, return None */ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { - if (str->ptr == NULL) { - return STRINGLIB_NEW(NULL, 0); + if (str->str == NULL) { + return PyUnicode_New(0, 0); } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); + return SubString_new_object(str); } /* Return 1 if an error has been detected switching between automatic @@ -121,74 +110,6 @@ autonumber_state_error(AutoNumberState state, int field_name_is_empty) /************************************************************************/ -/*********** Output string management functions ****************/ -/************************************************************************/ - -typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; - PyObject *obj; - Py_ssize_t size_increment; -} OutputString; - -/* initialize an OutputString object, reserving size characters */ -static int -output_initialize(OutputString *output, Py_ssize_t size) -{ - output->obj = STRINGLIB_NEW(NULL, size); - if (output->obj == NULL) - return 0; - - output->ptr = STRINGLIB_STR(output->obj); - output->end = STRINGLIB_LEN(output->obj) + output->ptr; - output->size_increment = INITIAL_SIZE_INCREMENT; - - return 1; -} - -/* - output_extend reallocates the output string buffer. - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ - -static int -output_extend(OutputString *output, Py_ssize_t count) -{ - STRINGLIB_CHAR *startptr = STRINGLIB_STR(output->obj); - Py_ssize_t curlen = output->ptr - startptr; - Py_ssize_t maxlen = curlen + count + output->size_increment; - - if (STRINGLIB_RESIZE(&output->obj, maxlen) < 0) - return 0; - startptr = STRINGLIB_STR(output->obj); - output->ptr = startptr + curlen; - output->end = startptr + maxlen; - if (output->size_increment < MAX_SIZE_INCREMENT) - output->size_increment *= SIZE_MULTIPLIER; - return 1; -} - -/* - output_data dumps characters into our output string - buffer. - - In some cases, it has to reallocate the string. - - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ -static int -output_data(OutputString *output, const STRINGLIB_CHAR *s, Py_ssize_t count) -{ - if ((count > output->end - output->ptr) && !output_extend(output, count)) - return 0; - memcpy(output->ptr, s, count * sizeof(STRINGLIB_CHAR)); - output->ptr += count; - return 1; -} - -/************************************************************************/ /*********** Format string parsing -- integers and identifiers *********/ /************************************************************************/ @@ -197,31 +118,28 @@ get_integer(const SubString *str) { Py_ssize_t accumulator = 0; Py_ssize_t digitval; - Py_ssize_t oldaccumulator; - STRINGLIB_CHAR *p; + Py_ssize_t i; /* empty string is an error */ - if (str->ptr >= str->end) + if (str->start >= str->end) return -1; - for (p = str->ptr; p < str->end; p++) { - digitval = STRINGLIB_TODECIMAL(*p); + for (i = str->start; i < str->end; i++) { + digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str->str, i)); if (digitval < 0) return -1; /* - This trick was copied from old Unicode format code. It's cute, - but would really suck on an old machine with a slow divide - implementation. Fortunately, in the normal case we do not - expect too many digits. + Detect possible overflow before it happens: + + accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if + accumulator > (PY_SSIZE_T_MAX - digitval) / 10. */ - oldaccumulator = accumulator; - accumulator *= 10; - if ((accumulator+10)/10 != oldaccumulator+1) { + if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { PyErr_Format(PyExc_ValueError, "Too many decimal digits in format string"); return -1; } - accumulator += digitval; + accumulator = accumulator * 10 + digitval; } return accumulator; } @@ -282,34 +200,36 @@ typedef struct { lifetime of the iterator. can be empty */ SubString str; - /* pointer to where we are inside field_name */ - STRINGLIB_CHAR *ptr; + /* index to where we are inside field_name */ + Py_ssize_t index; } FieldNameIterator; static int -FieldNameIterator_init(FieldNameIterator *self, STRINGLIB_CHAR *ptr, - Py_ssize_t len) +FieldNameIterator_init(FieldNameIterator *self, PyObject *s, + Py_ssize_t start, Py_ssize_t end) { - SubString_init(&self->str, ptr, len); - self->ptr = self->str.ptr; + SubString_init(&self->str, s, start, end); + self->index = start; return 1; } static int _FieldNameIterator_attr(FieldNameIterator *self, SubString *name) { - STRINGLIB_CHAR c; + Py_UCS4 c; - name->ptr = self->ptr; + name->str = self->str.str; + name->start = self->index; /* return everything until '.' or '[' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { case '[': case '.': /* backup so that we this character will be seen next time */ - self->ptr--; + self->index--; break; default: continue; @@ -317,7 +237,7 @@ _FieldNameIterator_attr(FieldNameIterator *self, SubString *name) break; } /* end of string is okay */ - name->end = self->ptr; + name->end = self->index; return 1; } @@ -325,13 +245,15 @@ static int _FieldNameIterator_item(FieldNameIterator *self, SubString *name) { int bracket_seen = 0; - STRINGLIB_CHAR c; + Py_UCS4 c; - name->ptr = self->ptr; + name->str = self->str.str; + name->start = self->index; /* return everything until ']' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { case ']': bracket_seen = 1; break; @@ -348,7 +270,7 @@ _FieldNameIterator_item(FieldNameIterator *self, SubString *name) /* end of string is okay */ /* don't include the ']' */ - name->end = self->ptr-1; + name->end = self->index-1; return 1; } @@ -358,10 +280,10 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, Py_ssize_t *name_idx, SubString *name) { /* check at end of input */ - if (self->ptr >= self->str.end) + if (self->index >= self->str.end) return 1; - switch (*self->ptr++) { + switch (PyUnicode_READ_CHAR(self->str.str, self->index++)) { case '.': *is_attribute = 1; if (_FieldNameIterator_attr(self, name) == 0) @@ -384,7 +306,7 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, } /* empty string is an error */ - if (name->ptr == name->end) { + if (name->start == name->end) { PyErr_SetString(PyExc_ValueError, "Empty attribute in format string"); return 0; } @@ -400,24 +322,23 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, 'rest' is an iterator to return the rest */ static int -field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first, +field_name_split(PyObject *str, Py_ssize_t start, Py_ssize_t end, SubString *first, Py_ssize_t *first_idx, FieldNameIterator *rest, AutoNumber *auto_number) { - STRINGLIB_CHAR c; - STRINGLIB_CHAR *p = ptr; - STRINGLIB_CHAR *end = ptr + len; + Py_UCS4 c; + Py_ssize_t i = start; int field_name_is_empty; int using_numeric_index; /* find the part up until the first '.' or '[' */ - while (p < end) { - switch (c = *p++) { + while (i < end) { + switch (c = PyUnicode_READ_CHAR(str, i++)) { case '[': case '.': /* backup so that we this character is available to the "rest" iterator */ - p--; + i--; break; default: continue; @@ -426,15 +347,15 @@ field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first, } /* set up the return values */ - SubString_init(first, ptr, p - ptr); - FieldNameIterator_init(rest, p, end - p); + SubString_init(first, str, start, i); + FieldNameIterator_init(rest, str, i, end); /* see if "first" is an integer, in which case it's used as an index */ *first_idx = get_integer(first); if (*first_idx == -1 && PyErr_Occurred()) return 0; - field_name_is_empty = first->ptr >= first->end; + field_name_is_empty = first->start >= first->end; /* If the field name is omitted or if we have a numeric index specified, then we're doing numeric indexing into args. */ @@ -489,7 +410,7 @@ get_field_object(SubString *input, PyObject *args, PyObject *kwargs, Py_ssize_t index; FieldNameIterator rest; - if (!field_name_split(input->ptr, input->end - input->ptr, &first, + if (!field_name_split(input->str, input->start, input->end, &first, &index, &rest, auto_number)) { goto error; } @@ -573,39 +494,41 @@ error: appends to the output. */ static int -render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) +render_field(PyObject *fieldobj, SubString *format_spec, _PyUnicodeWriter *writer) { int ok = 0; PyObject *result = NULL; PyObject *format_spec_object = NULL; - PyObject *(*formatter)(PyObject *, STRINGLIB_CHAR *, Py_ssize_t) = NULL; - STRINGLIB_CHAR* format_spec_start = format_spec->ptr ? - format_spec->ptr : NULL; - Py_ssize_t format_spec_len = format_spec->ptr ? - format_spec->end - format_spec->ptr : 0; + int (*formatter) (_PyUnicodeWriter*, PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; + int err; /* If we know the type exactly, skip the lookup of __format__ and just call the formatter directly. */ if (PyUnicode_CheckExact(fieldobj)) - formatter = _PyUnicode_FormatAdvanced; + formatter = _PyUnicode_FormatAdvancedWriter; else if (PyLong_CheckExact(fieldobj)) - formatter =_PyLong_FormatAdvanced; + formatter = _PyLong_FormatAdvancedWriter; else if (PyFloat_CheckExact(fieldobj)) - formatter = _PyFloat_FormatAdvanced; - - /* XXX: for 2.6, convert format_spec to the appropriate type - (unicode, str) */ + formatter = _PyFloat_FormatAdvancedWriter; + else if (PyComplex_CheckExact(fieldobj)) + formatter = _PyComplex_FormatAdvancedWriter; if (formatter) { /* we know exactly which formatter will be called when __format__ is looked up, so call it directly, instead. */ - result = formatter(fieldobj, format_spec_start, format_spec_len); + err = formatter(writer, fieldobj, format_spec->str, + format_spec->start, format_spec->end); + return (err == 0); } else { /* We need to create an object out of the pointers we have, because __format__ takes a string/unicode object for format_spec. */ - format_spec_object = STRINGLIB_NEW(format_spec_start, - format_spec_len); + if (format_spec->str) + format_spec_object = PyUnicode_Substring(format_spec->str, + format_spec->start, + format_spec->end); + else + format_spec_object = PyUnicode_New(0, 0); if (format_spec_object == NULL) goto done; @@ -614,24 +537,10 @@ render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) if (result == NULL) goto done; -#if PY_VERSION_HEX >= 0x03000000 - assert(PyUnicode_Check(result)); -#else - assert(PyBytes_Check(result) || PyUnicode_Check(result)); - - /* Convert result to our type. We could be str, and result could - be unicode */ - { - PyObject *tmp = STRINGLIB_TOSTR(result); - if (tmp == NULL) - goto done; - Py_DECREF(result); - result = tmp; - } -#endif + if (_PyUnicodeWriter_WriteStr(writer, result) == -1) + goto done; + ok = 1; - ok = output_data(output, - STRINGLIB_STR(result), STRINGLIB_LEN(result)); done: Py_XDECREF(format_spec_object); Py_XDECREF(result); @@ -640,23 +549,24 @@ done: static int parse_field(SubString *str, SubString *field_name, SubString *format_spec, - STRINGLIB_CHAR *conversion) + Py_UCS4 *conversion) { /* Note this function works if the field name is zero length, which is good. Zero length field names are handled later, in field_name_split. */ - STRINGLIB_CHAR c = 0; + Py_UCS4 c = 0; /* initialize these, as they may be empty */ *conversion = '\0'; - SubString_init(format_spec, NULL, 0); + SubString_init(format_spec, NULL, 0, 0); /* Search for the field name. it's terminated by the end of the string, or a ':' or '!' */ - field_name->ptr = str->ptr; - while (str->ptr < str->end) { - switch (c = *(str->ptr++)) { + field_name->str = str->str; + field_name->start = str->start; + while (str->start < str->end) { + switch ((c = PyUnicode_READ_CHAR(str->str, str->start++))) { case ':': case '!': break; @@ -669,26 +579,27 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec, if (c == '!' || c == ':') { /* we have a format specifier and/or a conversion */ /* don't include the last character */ - field_name->end = str->ptr-1; + field_name->end = str->start-1; /* the format specifier is the rest of the string */ - format_spec->ptr = str->ptr; + format_spec->str = str->str; + format_spec->start = str->start; format_spec->end = str->end; /* see if there's a conversion specifier */ if (c == '!') { /* there must be another character present */ - if (format_spec->ptr >= format_spec->end) { + if (format_spec->start >= format_spec->end) { PyErr_SetString(PyExc_ValueError, "end of format while looking for conversion " "specifier"); return 0; } - *conversion = *(format_spec->ptr++); + *conversion = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); /* if there is another character, it must be a colon */ - if (format_spec->ptr < format_spec->end) { - c = *(format_spec->ptr++); + if (format_spec->start < format_spec->end) { + c = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); if (c != ':') { PyErr_SetString(PyExc_ValueError, "expected ':' after format specifier"); @@ -699,7 +610,7 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec, } else /* end of string, there's no format_spec or conversion */ - field_name->end = str->ptr; + field_name->end = str->start; return 1; } @@ -718,9 +629,10 @@ typedef struct { } MarkupIterator; static int -MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len) +MarkupIterator_init(MarkupIterator *self, PyObject *str, + Py_ssize_t start, Py_ssize_t end) { - SubString_init(&self->str, ptr, len); + SubString_init(&self->str, str, start, end); return 1; } @@ -729,30 +641,30 @@ MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len) static int MarkupIterator_next(MarkupIterator *self, SubString *literal, int *field_present, SubString *field_name, - SubString *format_spec, STRINGLIB_CHAR *conversion, + SubString *format_spec, Py_UCS4 *conversion, int *format_spec_needs_expanding) { int at_end; - STRINGLIB_CHAR c = 0; - STRINGLIB_CHAR *start; + Py_UCS4 c = 0; + Py_ssize_t start; int count; Py_ssize_t len; int markup_follows = 0; /* initialize all of the output variables */ - SubString_init(literal, NULL, 0); - SubString_init(field_name, NULL, 0); - SubString_init(format_spec, NULL, 0); + SubString_init(literal, NULL, 0, 0); + SubString_init(field_name, NULL, 0, 0); + SubString_init(format_spec, NULL, 0, 0); *conversion = '\0'; *format_spec_needs_expanding = 0; *field_present = 0; /* No more input, end of iterator. This is the normal exit path. */ - if (self->str.ptr >= self->str.end) + if (self->str.start >= self->str.end) return 1; - start = self->str.ptr; + start = self->str.start; /* First read any literal text. Read until the end of string, an escaped '{' or '}', or an unescaped '{'. In order to never @@ -761,8 +673,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, including the brace, but no format object. The next time through, we'll return the rest of the literal, skipping past the second consecutive brace. */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { case '{': case '}': markup_follows = 1; @@ -773,10 +685,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, break; } - at_end = self->str.ptr >= self->str.end; - len = self->str.ptr - start; + at_end = self->str.start >= self->str.end; + len = self->str.start - start; - if ((c == '}') && (at_end || (c != *self->str.ptr))) { + if ((c == '}') && (at_end || + (c != PyUnicode_READ_CHAR(self->str.str, + self->str.start)))) { PyErr_SetString(PyExc_ValueError, "Single '}' encountered " "in format string"); return 0; @@ -787,10 +701,10 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, return 0; } if (!at_end) { - if (c == *self->str.ptr) { + if (c == PyUnicode_READ_CHAR(self->str.str, self->str.start)) { /* escaped } or {, skip it in the input. there is no markup object following us, just this literal text */ - self->str.ptr++; + self->str.start++; markup_follows = 0; } else @@ -798,7 +712,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, } /* record the literal text */ - literal->ptr = start; + literal->str = self->str.str; + literal->start = start; literal->end = start + len; if (!markup_follows) @@ -810,12 +725,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, *field_present = 1; count = 1; - start = self->str.ptr; + start = self->str.start; /* we know we can't have a zero length string, so don't worry about that case */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { case '{': /* the format spec needs to be recursively expanded. this is an optimization, and not strictly needed */ @@ -828,7 +743,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, /* we're done. parse and get out */ SubString s; - SubString_init(&s, start, self->str.ptr - 1 - start); + SubString_init(&s, self->str.str, start, self->str.start - 1); if (parse_field(&s, field_name, format_spec, conversion) == 0) return 0; @@ -847,7 +762,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, /* do the !r or !s conversion on obj */ static PyObject * -do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) +do_conversion(PyObject *obj, Py_UCS4 conversion) { /* XXX in pre-3.0, do we need to convert this to unicode, since it might have returned a string? */ @@ -855,11 +770,9 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) case 'r': return PyObject_Repr(obj); case 's': - return STRINGLIB_TOSTR(obj); -#if PY_VERSION_HEX >= 0x03000000 + return PyObject_Str(obj); case 'a': - return STRINGLIB_TOASCII(obj); -#endif + return PyObject_ASCII(obj); default: if (conversion > 32 && conversion < 127) { /* It's the ASCII subrange; casting to char is safe @@ -891,8 +804,8 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) static int output_markup(SubString *field_name, SubString *format_spec, - int format_spec_needs_expanding, STRINGLIB_CHAR conversion, - OutputString *output, PyObject *args, PyObject *kwargs, + int format_spec_needs_expanding, Py_UCS4 conversion, + _PyUnicodeWriter *writer, PyObject *args, PyObject *kwargs, int recursion_depth, AutoNumber *auto_number) { PyObject *tmp = NULL; @@ -908,7 +821,7 @@ output_markup(SubString *field_name, SubString *format_spec, if (conversion != '\0') { tmp = do_conversion(fieldobj, conversion); - if (tmp == NULL) + if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; /* do the assignment, transferring ownership: fieldobj = tmp */ @@ -921,20 +834,19 @@ output_markup(SubString *field_name, SubString *format_spec, if (format_spec_needs_expanding) { tmp = build_string(format_spec, args, kwargs, recursion_depth-1, auto_number); - if (tmp == NULL) + if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; /* note that in the case we're expanding the format string, tmp must be kept around until after the call to render_field. */ - SubString_init(&expanded_format_spec, - STRINGLIB_STR(tmp), STRINGLIB_LEN(tmp)); + SubString_init(&expanded_format_spec, tmp, 0, PyUnicode_GET_LENGTH(tmp)); actual_format_spec = &expanded_format_spec; } else actual_format_spec = format_spec; - if (render_field(fieldobj, actual_format_spec, output) == 0) + if (render_field(fieldobj, actual_format_spec, writer) == 0) goto done; result = 1; @@ -954,7 +866,7 @@ done: */ static int do_markup(SubString *input, PyObject *args, PyObject *kwargs, - OutputString *output, int recursion_depth, AutoNumber *auto_number) + _PyUnicodeWriter *writer, int recursion_depth, AutoNumber *auto_number) { MarkupIterator iter; int format_spec_needs_expanding; @@ -963,20 +875,35 @@ do_markup(SubString *input, PyObject *args, PyObject *kwargs, SubString literal; SubString field_name; SubString format_spec; - STRINGLIB_CHAR conversion; + Py_UCS4 conversion, maxchar; + Py_ssize_t sublen; + int err; - MarkupIterator_init(&iter, input->ptr, input->end - input->ptr); + MarkupIterator_init(&iter, input->str, input->start, input->end); while ((result = MarkupIterator_next(&iter, &literal, &field_present, &field_name, &format_spec, &conversion, &format_spec_needs_expanding)) == 2) { - if (!output_data(output, literal.ptr, literal.end - literal.ptr)) - return 0; - if (field_present) + sublen = literal.end - literal.start; + if (sublen) { + maxchar = _PyUnicode_FindMaxChar(literal.str, + literal.start, literal.end); + err = _PyUnicodeWriter_Prepare(writer, sublen, maxchar); + if (err == -1) + return 0; + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + literal.str, literal.start, sublen); + writer->pos += sublen; + } + + if (field_present) { + if (iter.str.start == iter.str.end) + writer->overallocate = 0; if (!output_markup(&field_name, &format_spec, - format_spec_needs_expanding, conversion, output, + format_spec_needs_expanding, conversion, writer, args, kwargs, recursion_depth, auto_number)) return 0; + } } return result; } @@ -990,43 +917,26 @@ static PyObject * build_string(SubString *input, PyObject *args, PyObject *kwargs, int recursion_depth, AutoNumber *auto_number) { - OutputString output; - PyObject *result = NULL; - Py_ssize_t count; - - output.obj = NULL; /* needed so cleanup code always works */ + _PyUnicodeWriter writer; + Py_ssize_t minlen; /* check the recursion level */ if (recursion_depth <= 0) { PyErr_SetString(PyExc_ValueError, "Max string recursion exceeded"); - goto done; + return NULL; } - /* initial size is the length of the format string, plus the size - increment. seems like a reasonable default */ - if (!output_initialize(&output, - input->end - input->ptr + - INITIAL_SIZE_INCREMENT)) - goto done; + minlen = PyUnicode_GET_LENGTH(input->str) + 100; + _PyUnicodeWriter_Init(&writer, minlen); - if (!do_markup(input, args, kwargs, &output, recursion_depth, + if (!do_markup(input, args, kwargs, &writer, recursion_depth, auto_number)) { - goto done; - } - - count = output.ptr - STRINGLIB_STR(output.obj); - if (STRINGLIB_RESIZE(&output.obj, count) < 0) { - goto done; + _PyUnicodeWriter_Dealloc(&writer); + return NULL; } - /* transfer ownership to result */ - result = output.obj; - output.obj = NULL; - -done: - Py_XDECREF(output.obj); - return result; + return _PyUnicodeWriter_Finish(&writer); } /************************************************************************/ @@ -1047,8 +957,11 @@ do_string_format(PyObject *self, PyObject *args, PyObject *kwargs) AutoNumber auto_number; + if (PyUnicode_READY(self) == -1) + return NULL; + AutoNumber_Init(&auto_number); - SubString_init(&input, STRINGLIB_STR(self), STRINGLIB_LEN(self)); + SubString_init(&input, self, 0, PyUnicode_GET_LENGTH(self)); return build_string(&input, args, kwargs, recursion_depth, &auto_number); } @@ -1070,9 +983,7 @@ do_string_format_map(PyObject *self, PyObject *obj) typedef struct { PyObject_HEAD - - STRINGLIB_OBJECT *str; - + PyObject *str; MarkupIterator it_markup; } formatteriterobject; @@ -1097,7 +1008,7 @@ formatteriter_next(formatteriterobject *it) SubString literal; SubString field_name; SubString format_spec; - STRINGLIB_CHAR conversion; + Py_UCS4 conversion; int format_spec_needs_expanding; int field_present; int result = MarkupIterator_next(&it->it_markup, &literal, &field_present, @@ -1141,7 +1052,8 @@ formatteriter_next(formatteriterobject *it) Py_INCREF(conversion_str); } else - conversion_str = STRINGLIB_NEW(&conversion, 1); + conversion_str = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + &conversion, 1); if (conversion_str == NULL) goto done; @@ -1198,7 +1110,7 @@ static PyTypeObject PyFormatterIter_Type = { describing the parsed elements. It's a wrapper around stringlib/string_format.h's MarkupIterator */ static PyObject * -formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) +formatter_parser(PyObject *ignored, PyObject *self) { formatteriterobject *it; @@ -1207,6 +1119,9 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) return NULL; } + if (PyUnicode_READY(self) == -1) + return NULL; + it = PyObject_New(formatteriterobject, &PyFormatterIter_Type); if (it == NULL) return NULL; @@ -1216,10 +1131,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) it->str = self; /* initialize the contained MarkupIterator */ - MarkupIterator_init(&it->it_markup, - STRINGLIB_STR(self), - STRINGLIB_LEN(self)); - + MarkupIterator_init(&it->it_markup, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)); return (PyObject *)it; } @@ -1235,9 +1147,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) typedef struct { PyObject_HEAD - - STRINGLIB_OBJECT *str; - + PyObject *str; FieldNameIterator it_field; } fieldnameiterobject; @@ -1338,7 +1248,7 @@ static PyTypeObject PyFieldNameIter_Type = { field_name_split. The iterator it returns is a FieldNameIterator */ static PyObject * -formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) +formatter_field_name_split(PyObject *ignored, PyObject *self) { SubString first; Py_ssize_t first_idx; @@ -1352,6 +1262,9 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) return NULL; } + if (PyUnicode_READY(self) == -1) + return NULL; + it = PyObject_New(fieldnameiterobject, &PyFieldNameIter_Type); if (it == NULL) return NULL; @@ -1363,8 +1276,7 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) /* Pass in auto_number = NULL. We'll return an empty string for first_obj in that case. */ - if (!field_name_split(STRINGLIB_STR(self), - STRINGLIB_LEN(self), + if (!field_name_split((PyObject*)self, 0, PyUnicode_GET_LENGTH(self), &first, &first_idx, &it->it_field, NULL)) goto done; diff --git a/Objects/stringlib/unicodedefs.h b/Objects/stringlib/unicodedefs.h index 09dae6dc9e..f16f21e60c 100644 --- a/Objects/stringlib/unicodedefs.h +++ b/Objects/stringlib/unicodedefs.h @@ -6,7 +6,10 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 1 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR Py_UNICODE_SIZE #define STRINGLIB_CHAR Py_UNICODE #define STRINGLIB_TYPE_NAME "unicode" #define STRINGLIB_PARSE_CODE "U" @@ -15,17 +18,12 @@ #define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK #define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL #define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL -#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER -#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER -#define STRINGLIB_FILL Py_UNICODE_FILL #define STRINGLIB_STR PyUnicode_AS_UNICODE #define STRINGLIB_LEN PyUnicode_GET_SIZE #define STRINGLIB_NEW PyUnicode_FromUnicode #define STRINGLIB_RESIZE PyUnicode_Resize #define STRINGLIB_CHECK PyUnicode_Check #define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact -#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping -#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale #if PY_VERSION_HEX < 0x03000000 #define STRINGLIB_TOSTR PyObject_Unicode |