summaryrefslogtreecommitdiff
path: root/Python/ast.c
diff options
context:
space:
mode:
Diffstat (limited to 'Python/ast.c')
-rw-r--r--Python/ast.c1678
1 files changed, 768 insertions, 910 deletions
diff --git a/Python/ast.c b/Python/ast.c
index fc6f002cde..d2f063bb5b 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -7,7 +7,6 @@
#include "Python-ast.h"
#include "grammar.h"
#include "node.h"
-#include "pyarena.h"
#include "ast.h"
#include "token.h"
#include "parsetok.h"
@@ -18,7 +17,6 @@
/* Data structure used internally */
struct compiling {
char *c_encoding; /* source encoding */
- int c_future_unicode; /* __future__ unicode literals flag */
PyArena *c_arena; /* arena for allocating memeory */
const char *c_filename; /* filename */
};
@@ -31,27 +29,50 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
-static expr_ty ast_for_testlist_comp(struct compiling *, const node *);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
static PyObject *parsenumber(struct compiling *, const char *);
-static PyObject *parsestr(struct compiling *, const char *);
-static PyObject *parsestrplus(struct compiling *, const node *n);
+static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode);
+static PyObject *parsestrplus(struct compiling *, const node *n,
+ int *bytesmode);
#ifndef LINENO
#define LINENO(n) ((n)->n_lineno)
#endif
-#define COMP_GENEXP 0
-#define COMP_SETCOMP 1
+#define COMP_GENEXP 0
+#define COMP_LISTCOMP 1
+#define COMP_SETCOMP 2
static identifier
-new_identifier(const char* n, PyArena *arena) {
- PyObject* id = PyString_InternFromString(n);
- if (id != NULL)
- PyArena_AddPyObject(arena, id);
+new_identifier(const char *n, PyArena *arena)
+{
+ PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
+ Py_UNICODE *u;
+ if (!id)
+ return NULL;
+ u = PyUnicode_AS_UNICODE(id);
+ /* Check whether there are non-ASCII characters in the
+ identifier; if so, normalize to NFKC. */
+ for (; *u; u++) {
+ if (*u >= 128) {
+ PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
+ PyObject *id2;
+ if (!m)
+ return NULL;
+ id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
+ Py_DECREF(m);
+ if (!id2)
+ return NULL;
+ Py_DECREF(id);
+ id = id2;
+ break;
+ }
+ }
+ PyUnicode_InternInPlace(&id);
+ PyArena_AddPyObject(arena, id);
return id;
}
@@ -68,10 +89,18 @@ new_identifier(const char* n, PyArena *arena) {
static int
ast_error(const node *n, const char *errstr)
{
- PyObject *u = Py_BuildValue("zi", errstr, LINENO(n));
+ PyObject *u = Py_BuildValue("zii", errstr, LINENO(n), n->n_col_offset), *save;
if (!u)
return 0;
+ /*
+ * Prevent the error from being chained. PyErr_SetObject will normalize the
+ * exception in order to chain it. ast_error_finish, however, requires the
+ * error not to be normalized.
+ */
+ save = PyThreadState_GET()->exc_value;
+ PyThreadState_GET()->exc_value = NULL;
PyErr_SetObject(PyExc_SyntaxError, u);
+ PyThreadState_GET()->exc_value = save;
Py_DECREF(u);
return 0;
}
@@ -79,7 +108,8 @@ ast_error(const node *n, const char *errstr)
static void
ast_error_finish(const char *filename)
{
- PyObject *type, *value, *tback, *errstr, *loc, *tmp;
+ PyObject *type, *value, *tback, *errstr, *offset, *loc, *tmp;
+ PyObject *filename_obj;
long lineno;
assert(PyErr_Occurred());
@@ -91,11 +121,16 @@ ast_error_finish(const char *filename)
if (!errstr)
return;
Py_INCREF(errstr);
- lineno = PyInt_AsLong(PyTuple_GetItem(value, 1));
+ lineno = PyLong_AsLong(PyTuple_GetItem(value, 1));
if (lineno == -1) {
Py_DECREF(errstr);
return;
}
+ offset = PyTuple_GetItem(value, 2);
+ if (!offset) {
+ Py_DECREF(errstr);
+ return;
+ }
Py_DECREF(value);
loc = PyErr_ProgramText(filename, lineno);
@@ -103,7 +138,16 @@ ast_error_finish(const char *filename)
Py_INCREF(Py_None);
loc = Py_None;
}
- tmp = Py_BuildValue("(zlOO)", filename, lineno, Py_None, loc);
+ if (filename != NULL)
+ filename_obj = PyUnicode_DecodeFSDefault(filename);
+ else {
+ Py_INCREF(Py_None);
+ filename_obj = Py_None;
+ }
+ if (filename_obj != NULL)
+ tmp = Py_BuildValue("(NlOO)", filename_obj, lineno, offset, loc);
+ else
+ tmp = NULL;
Py_DECREF(loc);
if (!tmp) {
Py_DECREF(errstr);
@@ -117,37 +161,6 @@ ast_error_finish(const char *filename)
PyErr_Restore(type, value, tback);
}
-static int
-ast_warn(struct compiling *c, const node *n, char *msg)
-{
- if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, c->c_filename, LINENO(n),
- NULL, NULL) < 0) {
- /* if -Werr, change it to a SyntaxError */
- if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SyntaxWarning))
- ast_error(n, msg);
- return 0;
- }
- return 1;
-}
-
-static int
-forbidden_check(struct compiling *c, const node *n, const char *x)
-{
- if (!strcmp(x, "None"))
- return ast_error(n, "cannot assign to None");
- if (!strcmp(x, "__debug__"))
- return ast_error(n, "cannot assign to __debug__");
- if (Py_Py3kWarningFlag) {
- if (!(strcmp(x, "True") && strcmp(x, "False")) &&
- !ast_warn(c, n, "assignment to True or False is forbidden in 3.x"))
- return 0;
- if (!strcmp(x, "nonlocal") &&
- !ast_warn(c, n, "nonlocal is a keyword in 3.x"))
- return 0;
- }
- return 1;
-}
-
/* num_stmts() returns number of contained statements.
Use this routine to determine how big a sequence is needed for
@@ -225,16 +238,19 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) {
c.c_encoding = "utf-8";
if (TYPE(n) == encoding_decl) {
+#if 0
ast_error(n, "encoding declaration in Unicode string");
goto error;
+#endif
+ n = CHILD(n, 0);
}
} else if (TYPE(n) == encoding_decl) {
c.c_encoding = STR(n);
n = CHILD(n, 0);
} else {
- c.c_encoding = NULL;
+ /* PEP 3120 */
+ c.c_encoding = "utf-8";
}
- c.c_future_unicode = flags && flags->cf_flags & CO_FUTURE_UNICODE_LITERALS;
c.c_arena = arena;
c.c_filename = filename;
@@ -359,6 +375,33 @@ get_operator(const node *n)
}
}
+static const char* FORBIDDEN[] = {
+ "None",
+ "True",
+ "False",
+ NULL,
+};
+
+static int
+forbidden_name(identifier name, const node *n, int full_checks)
+{
+ assert(PyUnicode_Check(name));
+ if (PyUnicode_CompareWithASCIIString(name, "__debug__") == 0) {
+ ast_error(n, "assignment to keyword");
+ return 1;
+ }
+ if (full_checks) {
+ const char **p;
+ for (p = FORBIDDEN; *p; p++) {
+ if (PyUnicode_CompareWithASCIIString(name, *p) == 0) {
+ ast_error(n, "assignment to keyword");
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
/* Set the context ctx for expr_ty e, recursively traversing e.
Only sets context for expr kinds that "can appear in assignment context"
@@ -386,18 +429,23 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
switch (e->kind) {
case Attribute_kind:
- if (ctx == Store && !forbidden_check(c, n,
- PyBytes_AS_STRING(e->v.Attribute.attr)))
- return 0;
e->v.Attribute.ctx = ctx;
+ if (ctx == Store && forbidden_name(e->v.Attribute.attr, n, 1))
+ return 0;
break;
case Subscript_kind:
e->v.Subscript.ctx = ctx;
break;
+ case Starred_kind:
+ e->v.Starred.ctx = ctx;
+ if (!set_context(c, e->v.Starred.value, ctx, n))
+ return 0;
+ break;
case Name_kind:
- if (ctx == Store && !forbidden_check(c, n,
- PyBytes_AS_STRING(e->v.Name.id)))
- return 0;
+ if (ctx == Store) {
+ if (forbidden_name(e->v.Name.id, n, 1))
+ return 0; /* forbidden_name() calls ast_error() */
+ }
e->v.Name.ctx = ctx;
break;
case List_kind:
@@ -443,14 +491,15 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
case Set_kind:
case Num_kind:
case Str_kind:
+ case Bytes_kind:
expr_name = "literal";
break;
+ case Ellipsis_kind:
+ expr_name = "Ellipsis";
+ break;
case Compare_kind:
expr_name = "comparison";
break;
- case Repr_kind:
- expr_name = "repr";
- break;
case IfExp_kind:
expr_name = "conditional expression";
break;
@@ -525,7 +574,7 @@ ast_for_augassign(struct compiling *c, const node *n)
static cmpop_ty
ast_for_comp_op(struct compiling *c, const node *n)
{
- /* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'
+ /* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'
|'is' 'not'
*/
REQ(n, comp_op);
@@ -577,24 +626,23 @@ ast_for_comp_op(struct compiling *c, const node *n)
static asdl_seq *
seq_for_testlist(struct compiling *c, const node *n)
{
- /* testlist: test (',' test)* [','] */
+ /* testlist: test (',' test)* [',']
+ testlist_star_expr: test|star_expr (',' test|star_expr)* [',']
+ */
asdl_seq *seq;
expr_ty expression;
int i;
- assert(TYPE(n) == testlist ||
- TYPE(n) == listmaker ||
- TYPE(n) == testlist_comp ||
- TYPE(n) == testlist_safe ||
- TYPE(n) == testlist1);
+ assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || TYPE(n) == testlist_comp);
seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!seq)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
- assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test);
+ const node *ch = CHILD(n, i);
+ assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr);
- expression = ast_for_expr(c, CHILD(n, i));
+ expression = ast_for_expr(c, ch);
if (!expression)
return NULL;
@@ -604,97 +652,189 @@ seq_for_testlist(struct compiling *c, const node *n)
return seq;
}
-static expr_ty
-compiler_complex_args(struct compiling *c, const node *n)
+static arg_ty
+ast_for_arg(struct compiling *c, const node *n)
{
- int i, len = (NCH(n) + 1) / 2;
- expr_ty result;
- asdl_seq *args = asdl_seq_new(len, c->c_arena);
- if (!args)
+ identifier name;
+ expr_ty annotation = NULL;
+ node *ch;
+
+ assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef);
+ ch = CHILD(n, 0);
+ name = NEW_IDENTIFIER(ch);
+ if (!name)
+ return NULL;
+ if (forbidden_name(name, ch, 0))
return NULL;
- /* fpdef: NAME | '(' fplist ')'
- fplist: fpdef (',' fpdef)* [',']
- */
- REQ(n, fplist);
- for (i = 0; i < len; i++) {
- PyObject *arg_id;
- const node *fpdef_node = CHILD(n, 2*i);
- const node *child;
- expr_ty arg;
-set_name:
- /* fpdef_node is either a NAME or an fplist */
- child = CHILD(fpdef_node, 0);
- if (TYPE(child) == NAME) {
- if (!forbidden_check(c, n, STR(child)))
- return NULL;
- arg_id = NEW_IDENTIFIER(child);
- if (!arg_id)
- return NULL;
- arg = Name(arg_id, Store, LINENO(child), child->n_col_offset,
- c->c_arena);
- }
- else {
- assert(TYPE(fpdef_node) == fpdef);
- /* fpdef_node[0] is not a name, so it must be '(', get CHILD[1] */
- child = CHILD(fpdef_node, 1);
- assert(TYPE(child) == fplist);
- /* NCH == 1 means we have (x), we need to elide the extra parens */
- if (NCH(child) == 1) {
- fpdef_node = CHILD(child, 0);
- assert(TYPE(fpdef_node) == fpdef);
- goto set_name;
- }
- arg = compiler_complex_args(c, child);
- }
- asdl_seq_SET(args, i, arg);
+ if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {
+ annotation = ast_for_expr(c, CHILD(n, 2));
+ if (!annotation)
+ return NULL;
}
- result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);
- if (!set_context(c, result, Store, n))
- return NULL;
- return result;
+ return arg(name, annotation, c->c_arena);
}
+/* returns -1 if failed to handle keyword only arguments
+ returns new position to keep processing if successful
+ (',' tfpdef ['=' test])*
+ ^^^
+ start pointing here
+ */
+static int
+handle_keywordonly_args(struct compiling *c, const node *n, int start,
+ asdl_seq *kwonlyargs, asdl_seq *kwdefaults)
+{
+ PyObject *argname;
+ node *ch;
+ expr_ty expression, annotation;
+ arg_ty arg;
+ int i = start;
+ int j = 0; /* index for kwdefaults and kwonlyargs */
+
+ if (kwonlyargs == NULL) {
+ ast_error(CHILD(n, start), "named arguments must follow bare *");
+ return -1;
+ }
+ assert(kwdefaults != NULL);
+ while (i < NCH(n)) {
+ ch = CHILD(n, i);
+ switch (TYPE(ch)) {
+ case vfpdef:
+ case tfpdef:
+ if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
+ expression = ast_for_expr(c, CHILD(n, i + 2));
+ if (!expression)
+ goto error;
+ asdl_seq_SET(kwdefaults, j, expression);
+ i += 2; /* '=' and test */
+ }
+ else { /* setting NULL if no default value exists */
+ asdl_seq_SET(kwdefaults, j, NULL);
+ }
+ if (NCH(ch) == 3) {
+ /* ch is NAME ':' test */
+ annotation = ast_for_expr(c, CHILD(ch, 2));
+ if (!annotation)
+ goto error;
+ }
+ else {
+ annotation = NULL;
+ }
+ ch = CHILD(ch, 0);
+ argname = NEW_IDENTIFIER(ch);
+ if (!argname)
+ goto error;
+ if (forbidden_name(argname, ch, 0))
+ goto error;
+ arg = arg(argname, annotation, c->c_arena);
+ if (!arg)
+ goto error;
+ asdl_seq_SET(kwonlyargs, j++, arg);
+ i += 2; /* the name and the comma */
+ break;
+ case DOUBLESTAR:
+ return i;
+ default:
+ ast_error(ch, "unexpected node");
+ goto error;
+ }
+ }
+ return i;
+ error:
+ return -1;
+}
/* Create AST for argument list. */
static arguments_ty
ast_for_arguments(struct compiling *c, const node *n)
{
- /* parameters: '(' [varargslist] ')'
- varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME]
- | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+ /* This function handles both typedargslist (function definition)
+ and varargslist (lambda definition).
+
+ parameters: '(' [typedargslist] ')'
+ typedargslist: ((tfpdef ['=' test] ',')*
+ ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef]
+ | '**' tfpdef)
+ | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+ tfpdef: NAME [':' test]
+ varargslist: ((vfpdef ['=' test] ',')*
+ ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef]
+ | '**' vfpdef)
+ | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+ vfpdef: NAME
*/
- int i, j, k, n_args = 0, n_defaults = 0, found_default = 0;
- asdl_seq *args, *defaults;
+ int i, j, k, nposargs = 0, nkwonlyargs = 0;
+ int nposdefaults = 0, found_default = 0;
+ asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
identifier vararg = NULL, kwarg = NULL;
+ arg_ty arg;
+ expr_ty varargannotation = NULL, kwargannotation = NULL;
node *ch;
if (TYPE(n) == parameters) {
if (NCH(n) == 2) /* () as argument list */
- return arguments(NULL, NULL, NULL, NULL, c->c_arena);
+ return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ NULL, c->c_arena);
n = CHILD(n, 1);
}
- REQ(n, varargslist);
+ assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);
- /* first count the number of normal args & defaults */
+ /* First count the number of positional args & defaults. The
+ variable i is the loop index for this for loop and the next.
+ The next loop picks up where the first leaves off.
+ */
for (i = 0; i < NCH(n); i++) {
ch = CHILD(n, i);
- if (TYPE(ch) == fpdef)
- n_args++;
- if (TYPE(ch) == EQUAL)
- n_defaults++;
+ if (TYPE(ch) == STAR) {
+ /* skip star */
+ i++;
+ if (i < NCH(n) && /* skip argument following star */
+ (TYPE(CHILD(n, i)) == tfpdef ||
+ TYPE(CHILD(n, i)) == vfpdef)) {
+ i++;
+ }
+ break;
+ }
+ if (TYPE(ch) == DOUBLESTAR) break;
+ if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
+ if (TYPE(ch) == EQUAL) nposdefaults++;
+ }
+ /* count the number of keyword only args &
+ defaults for keyword only args */
+ for ( ; i < NCH(n); ++i) {
+ ch = CHILD(n, i);
+ if (TYPE(ch) == DOUBLESTAR) break;
+ if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;
}
- args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL);
- if (!args && n_args)
+ posargs = (nposargs ? asdl_seq_new(nposargs, c->c_arena) : NULL);
+ if (!posargs && nposargs)
+ return NULL;
+ kwonlyargs = (nkwonlyargs ?
+ asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
+ if (!kwonlyargs && nkwonlyargs)
return NULL;
- defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL);
- if (!defaults && n_defaults)
+ posdefaults = (nposdefaults ?
+ asdl_seq_new(nposdefaults, c->c_arena) : NULL);
+ if (!posdefaults && nposdefaults)
return NULL;
+ /* The length of kwonlyargs and kwdefaults are same
+ since we set NULL as default for keyword only argument w/o default
+ - we have sequence data structure, but no dictionary */
+ kwdefaults = (nkwonlyargs ?
+ asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
+ if (!kwdefaults && nkwonlyargs)
+ return NULL;
+
+ if (nposargs + nkwonlyargs > 255) {
+ ast_error(n, "more than 255 arguments");
+ return NULL;
+ }
- /* fpdef: NAME | '(' fplist ')'
- fplist: fpdef (',' fpdef)* [',']
+ /* tfpdef: NAME [':' test]
+ vfpdef: NAME
*/
i = 0;
j = 0; /* index for defaults */
@@ -702,9 +842,8 @@ ast_for_arguments(struct compiling *c, const node *n)
while (i < NCH(n)) {
ch = CHILD(n, i);
switch (TYPE(ch)) {
- case fpdef: {
- int complex_args = 0, parenthesized = 0;
- handle_fpdef:
+ case tfpdef:
+ case vfpdef:
/* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
anything other than EQUAL or a comma? */
/* XXX Should NCH(n) check be made a separate check? */
@@ -712,81 +851,74 @@ ast_for_arguments(struct compiling *c, const node *n)
expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression)
return NULL;
- assert(defaults != NULL);
- asdl_seq_SET(defaults, j++, expression);
+ assert(posdefaults != NULL);
+ asdl_seq_SET(posdefaults, j++, expression);
i += 2;
found_default = 1;
}
else if (found_default) {
- /* def f((x)=4): pass should raise an error.
- def f((x, (y))): pass will just incur the tuple unpacking warning. */
- if (parenthesized && !complex_args) {
- ast_error(n, "parenthesized arg with default");
- return NULL;
- }
ast_error(n,
"non-default argument follows default argument");
return NULL;
}
- if (NCH(ch) == 3) {
- ch = CHILD(ch, 1);
- /* def foo((x)): is not complex, special case. */
- if (NCH(ch) != 1) {
- /* We have complex arguments, setup for unpacking. */
- if (Py_Py3kWarningFlag && !ast_warn(c, ch,
- "tuple parameter unpacking has been removed in 3.x"))
- return NULL;
- complex_args = 1;
- asdl_seq_SET(args, k++, compiler_complex_args(c, ch));
- if (!asdl_seq_GET(args, k-1))
- return NULL;
- } else {
- /* def foo((x)): setup for checking NAME below. */
- /* Loop because there can be many parens and tuple
- unpacking mixed in. */
- parenthesized = 1;
- ch = CHILD(ch, 0);
- assert(TYPE(ch) == fpdef);
- goto handle_fpdef;
- }
- }
- if (TYPE(CHILD(ch, 0)) == NAME) {
- PyObject *id;
- expr_ty name;
- if (!forbidden_check(c, n, STR(CHILD(ch, 0))))
- return NULL;
- id = NEW_IDENTIFIER(CHILD(ch, 0));
- if (!id)
- return NULL;
- name = Name(id, Param, LINENO(ch), ch->n_col_offset,
- c->c_arena);
- if (!name)
- return NULL;
- asdl_seq_SET(args, k++, name);
-
- }
- i += 2; /* the name and the comma */
- if (parenthesized && Py_Py3kWarningFlag &&
- !ast_warn(c, ch, "parenthesized argument names "
- "are invalid in 3.x"))
+ arg = ast_for_arg(c, ch);
+ if (!arg)
return NULL;
-
+ asdl_seq_SET(posargs, k++, arg);
+ i += 2; /* the name and the comma */
break;
- }
case STAR:
- if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
+ if (i+1 >= NCH(n)) {
+ ast_error(CHILD(n, i),
+ "named arguments must follow bare *");
return NULL;
- vararg = NEW_IDENTIFIER(CHILD(n, i+1));
- if (!vararg)
- return NULL;
- i += 3;
+ }
+ ch = CHILD(n, i+1); /* tfpdef or COMMA */
+ if (TYPE(ch) == COMMA) {
+ int res = 0;
+ i += 2; /* now follows keyword only arguments */
+ res = handle_keywordonly_args(c, n, i,
+ kwonlyargs, kwdefaults);
+ if (res == -1) return NULL;
+ i = res; /* res has new position to process */
+ }
+ else {
+ vararg = NEW_IDENTIFIER(CHILD(ch, 0));
+ if (!vararg)
+ return NULL;
+ if (forbidden_name(vararg, CHILD(ch, 0), 0))
+ return NULL;
+ if (NCH(ch) > 1) {
+ /* there is an annotation on the vararg */
+ varargannotation = ast_for_expr(c, CHILD(ch, 2));
+ if (!varargannotation)
+ return NULL;
+ }
+ i += 3;
+ if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef
+ || TYPE(CHILD(n, i)) == vfpdef)) {
+ int res = 0;
+ res = handle_keywordonly_args(c, n, i,
+ kwonlyargs, kwdefaults);
+ if (res == -1) return NULL;
+ i = res; /* res has new position to process */
+ }
+ }
break;
case DOUBLESTAR:
- if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
- return NULL;
- kwarg = NEW_IDENTIFIER(CHILD(n, i+1));
+ ch = CHILD(n, i+1); /* tfpdef */
+ assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
+ kwarg = NEW_IDENTIFIER(CHILD(ch, 0));
if (!kwarg)
return NULL;
+ if (NCH(ch) > 1) {
+ /* there is an annotation on the kwarg */
+ kwargannotation = ast_for_expr(c, CHILD(ch, 2));
+ if (!kwargannotation)
+ return NULL;
+ }
+ if (forbidden_name(kwarg, CHILD(ch, 0), 0))
+ return NULL;
i += 3;
break;
default:
@@ -796,8 +928,8 @@ ast_for_arguments(struct compiling *c, const node *n)
return NULL;
}
}
-
- return arguments(args, vararg, kwarg, defaults, c->c_arena);
+ return arguments(posargs, vararg, varargannotation, kwonlyargs, kwarg,
+ kwargannotation, posdefaults, kwdefaults, c->c_arena);
}
static expr_ty
@@ -892,10 +1024,11 @@ ast_for_decorators(struct compiling *c, const node *n)
static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
- /* funcdef: 'def' NAME parameters ':' suite */
+ /* funcdef: 'def' NAME parameters ['->' test] ':' suite */
identifier name;
arguments_ty args;
asdl_seq *body;
+ expr_ty returns = NULL;
int name_i = 1;
REQ(n, funcdef);
@@ -903,16 +1036,22 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
name = NEW_IDENTIFIER(CHILD(n, name_i));
if (!name)
return NULL;
- else if (!forbidden_check(c, CHILD(n, name_i), STR(CHILD(n, name_i))))
+ if (forbidden_name(name, CHILD(n, name_i), 0))
return NULL;
args = ast_for_arguments(c, CHILD(n, name_i + 1));
if (!args)
return NULL;
+ if (TYPE(CHILD(n, name_i+2)) == RARROW) {
+ returns = ast_for_expr(c, CHILD(n, name_i + 3));
+ if (!returns)
+ return NULL;
+ name_i += 2;
+ }
body = ast_for_suite(c, CHILD(n, name_i + 3));
if (!body)
return NULL;
- return FunctionDef(name, args, body, decorator_seq, LINENO(n),
+ return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n),
n->n_col_offset, c->c_arena);
}
@@ -938,7 +1077,7 @@ ast_for_decorated(struct compiling *c, const node *n)
thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);
}
/* we count the decorators in when talking about the class' or
- function's line number */
+ * function's line number */
if (thing) {
thing->lineno = LINENO(n);
thing->col_offset = n->n_col_offset;
@@ -949,12 +1088,14 @@ ast_for_decorated(struct compiling *c, const node *n)
static expr_ty
ast_for_lambdef(struct compiling *c, const node *n)
{
- /* lambdef: 'lambda' [varargslist] ':' test */
+ /* lambdef: 'lambda' [varargslist] ':' test
+ lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
arguments_ty args;
expr_ty expression;
if (NCH(n) == 3) {
- args = arguments(NULL, NULL, NULL, NULL, c->c_arena);
+ args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ NULL, c->c_arena);
if (!args)
return NULL;
expression = ast_for_expr(c, CHILD(n, 2));
@@ -993,168 +1134,6 @@ ast_for_ifexpr(struct compiling *c, const node *n)
c->c_arena);
}
-/* XXX(nnorwitz): the listcomp and genexpr code should be refactored
- so there is only a single version. Possibly for loops can also re-use
- the code.
-*/
-
-/* Count the number of 'for' loop in a list comprehension.
-
- Helper for ast_for_listcomp().
-*/
-
-static int
-count_list_fors(struct compiling *c, const node *n)
-{
- int n_fors = 0;
- node *ch = CHILD(n, 1);
-
- count_list_for:
- n_fors++;
- REQ(ch, list_for);
- if (NCH(ch) == 5)
- ch = CHILD(ch, 4);
- else
- return n_fors;
- count_list_iter:
- REQ(ch, list_iter);
- ch = CHILD(ch, 0);
- if (TYPE(ch) == list_for)
- goto count_list_for;
- else if (TYPE(ch) == list_if) {
- if (NCH(ch) == 3) {
- ch = CHILD(ch, 2);
- goto count_list_iter;
- }
- else
- return n_fors;
- }
-
- /* Should never be reached */
- PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");
- return -1;
-}
-
-/* Count the number of 'if' statements in a list comprehension.
-
- Helper for ast_for_listcomp().
-*/
-
-static int
-count_list_ifs(struct compiling *c, const node *n)
-{
- int n_ifs = 0;
-
- count_list_iter:
- REQ(n, list_iter);
- if (TYPE(CHILD(n, 0)) == list_for)
- return n_ifs;
- n = CHILD(n, 0);
- REQ(n, list_if);
- n_ifs++;
- if (NCH(n) == 2)
- return n_ifs;
- n = CHILD(n, 2);
- goto count_list_iter;
-}
-
-static expr_ty
-ast_for_listcomp(struct compiling *c, const node *n)
-{
- /* listmaker: test ( list_for | (',' test)* [','] )
- list_for: 'for' exprlist 'in' testlist_safe [list_iter]
- list_iter: list_for | list_if
- list_if: 'if' test [list_iter]
- testlist_safe: test [(',' test)+ [',']]
- */
- expr_ty elt, first;
- asdl_seq *listcomps;
- int i, n_fors;
- node *ch;
-
- REQ(n, listmaker);
- assert(NCH(n) > 1);
-
- elt = ast_for_expr(c, CHILD(n, 0));
- if (!elt)
- return NULL;
-
- n_fors = count_list_fors(c, n);
- if (n_fors == -1)
- return NULL;
-
- listcomps = asdl_seq_new(n_fors, c->c_arena);
- if (!listcomps)
- return NULL;
-
- ch = CHILD(n, 1);
- for (i = 0; i < n_fors; i++) {
- comprehension_ty lc;
- asdl_seq *t;
- expr_ty expression;
- node *for_ch;
-
- REQ(ch, list_for);
-
- for_ch = CHILD(ch, 1);
- t = ast_for_exprlist(c, for_ch, Store);
- if (!t)
- return NULL;
- expression = ast_for_testlist(c, CHILD(ch, 3));
- if (!expression)
- return NULL;
-
- /* Check the # of children rather than the length of t, since
- [x for x, in ... ] has 1 element in t, but still requires a Tuple.
- */
- first = (expr_ty)asdl_seq_GET(t, 0);
- if (NCH(for_ch) == 1)
- lc = comprehension(first, expression, NULL, c->c_arena);
- else
- lc = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
- c->c_arena),
- expression, NULL, c->c_arena);
- if (!lc)
- return NULL;
-
- if (NCH(ch) == 5) {
- int j, n_ifs;
- asdl_seq *ifs;
- expr_ty list_for_expr;
-
- ch = CHILD(ch, 4);
- n_ifs = count_list_ifs(c, ch);
- if (n_ifs == -1)
- return NULL;
-
- ifs = asdl_seq_new(n_ifs, c->c_arena);
- if (!ifs)
- return NULL;
-
- for (j = 0; j < n_ifs; j++) {
- REQ(ch, list_iter);
- ch = CHILD(ch, 0);
- REQ(ch, list_if);
-
- list_for_expr = ast_for_expr(c, CHILD(ch, 1));
- if (!list_for_expr)
- return NULL;
-
- asdl_seq_SET(ifs, j, list_for_expr);
- if (NCH(ch) == 3)
- ch = CHILD(ch, 2);
- }
- /* on exit, must guarantee that ch is a list_for */
- if (TYPE(ch) == list_iter)
- ch = CHILD(ch, 0);
- lc->ifs = ifs;
- }
- asdl_seq_SET(listcomps, i, lc);
- }
-
- return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
-}
-
/*
Count the number of 'for' loops in a comprehension.
@@ -1296,6 +1275,8 @@ ast_for_comprehension(struct compiling *c, const node *n)
static expr_ty
ast_for_itercomp(struct compiling *c, const node *n, int type)
{
+ /* testlist_comp: test ( comp_for | (',' test)* [','] )
+ argument: [test '='] test [comp_for] # Really [keyword '='] test */
expr_ty elt;
asdl_seq *comps;
@@ -1311,6 +1292,8 @@ ast_for_itercomp(struct compiling *c, const node *n, int type)
if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+ else if (type == COMP_LISTCOMP)
+ return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else
@@ -1330,7 +1313,6 @@ ast_for_dictcomp(struct compiling *c, const node *n)
key = ast_for_expr(c, CHILD(n, 0));
if (!key)
return NULL;
-
value = ast_for_expr(c, CHILD(n, 2));
if (!value)
return NULL;
@@ -1350,19 +1332,29 @@ ast_for_genexp(struct compiling *c, const node *n)
}
static expr_ty
+ast_for_listcomp(struct compiling *c, const node *n)
+{
+ assert(TYPE(n) == (testlist_comp));
+ return ast_for_itercomp(c, n, COMP_LISTCOMP);
+}
+
+static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_itercomp(c, n, COMP_SETCOMP);
}
+
static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
- /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']'
- | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
+ /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
+ | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
+ | '...' | 'None' | 'True' | 'False'
*/
node *ch = CHILD(n, 0);
+ int bytesmode = 0;
switch (TYPE(ch)) {
case NAME: {
@@ -1374,17 +1366,16 @@ ast_for_atom(struct compiling *c, const node *n)
return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
case STRING: {
- PyObject *str = parsestrplus(c, n);
+ PyObject *str = parsestrplus(c, n, &bytesmode);
if (!str) {
-#ifdef Py_USING_UNICODE
- if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
+ if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
PyObject *type, *value, *tback, *errstr;
PyErr_Fetch(&type, &value, &tback);
errstr = PyObject_Str(value);
if (errstr) {
char *s = "";
char buf[128];
- s = PyString_AsString(errstr);
+ s = _PyUnicode_AsString(errstr);
PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
ast_error(n, buf);
Py_DECREF(errstr);
@@ -1395,11 +1386,13 @@ ast_for_atom(struct compiling *c, const node *n)
Py_DECREF(value);
Py_XDECREF(tback);
}
-#endif
return NULL;
}
PyArena_AddPyObject(c->c_arena, str);
- return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
+ if (bytesmode)
+ return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena);
+ else
+ return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
}
case NUMBER: {
PyObject *pynum = parsenumber(c, STR(ch));
@@ -1409,6 +1402,8 @@ ast_for_atom(struct compiling *c, const node *n)
PyArena_AddPyObject(c->c_arena, pynum);
return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
}
+ case ELLIPSIS: /* Ellipsis */
+ return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
@@ -1418,14 +1413,18 @@ ast_for_atom(struct compiling *c, const node *n)
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
- return ast_for_testlist_comp(c, ch);
+ /* testlist_comp: test ( comp_for | (',' test)* [','] ) */
+ if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for))
+ return ast_for_genexp(c, ch);
+
+ return ast_for_testlist(c, ch);
case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
- REQ(ch, listmaker);
+ REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts)
@@ -1436,10 +1435,8 @@ ast_for_atom(struct compiling *c, const node *n)
else
return ast_for_listcomp(c, ch);
case LBRACE: {
- /* dictorsetmaker:
- * (test ':' test (comp_for | (',' test ':' test)* [','])) |
- * (test (comp_for | (',' test)* [',']))
- */
+ /* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
+ * test (gen_for | (',' test)* [',']) */
int i, size;
asdl_seq *keys, *values;
@@ -1496,17 +1493,6 @@ ast_for_atom(struct compiling *c, const node *n)
return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
}
}
- case BACKQUOTE: { /* repr */
- expr_ty expression;
- if (Py_Py3kWarningFlag &&
- !ast_warn(c, n, "backquote not supported in 3.x; use repr()"))
- return NULL;
- expression = ast_for_testlist(c, CHILD(n, 1));
- if (!expression)
- return NULL;
-
- return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena);
- }
default:
PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
return NULL;
@@ -1522,13 +1508,10 @@ ast_for_slice(struct compiling *c, const node *n)
REQ(n, subscript);
/*
- subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
+ subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
*/
ch = CHILD(n, 0);
- if (TYPE(ch) == DOT)
- return Ellipsis(c->c_arena);
-
if (NCH(n) == 1 && TYPE(ch) == test) {
/* 'step' variable hold no significance in terms of being used over
other vars */
@@ -1568,21 +1551,7 @@ ast_for_slice(struct compiling *c, const node *n)
ch = CHILD(n, NCH(n) - 1);
if (TYPE(ch) == sliceop) {
- if (NCH(ch) == 1) {
- /*
- This is an extended slice (ie "x[::]") with no expression in the
- step field. We set this literally to "None" in order to
- disambiguate it from x[:]. (The interpreter might have to call
- __getslice__ for x[:], but it must call __getitem__ for x[::].)
- */
- identifier none = new_identifier("None", c->c_arena);
- if (!none)
- return NULL;
- ch = CHILD(ch, 0);
- step = Name(none, Load, LINENO(ch), ch->n_col_offset, c->c_arena);
- if (!step)
- return NULL;
- } else {
+ if (NCH(ch) != 1) {
ch = CHILD(ch, 1);
if (TYPE(ch) == test) {
step = ast_for_expr(c, ch);
@@ -1598,53 +1567,53 @@ ast_for_slice(struct compiling *c, const node *n)
static expr_ty
ast_for_binop(struct compiling *c, const node *n)
{
- /* Must account for a sequence of expressions.
- How should A op B op C by represented?
- BinOp(BinOp(A, op, B), op, C).
- */
+ /* Must account for a sequence of expressions.
+ How should A op B op C by represented?
+ BinOp(BinOp(A, op, B), op, C).
+ */
- int i, nops;
- expr_ty expr1, expr2, result;
- operator_ty newoperator;
+ int i, nops;
+ expr_ty expr1, expr2, result;
+ operator_ty newoperator;
- expr1 = ast_for_expr(c, CHILD(n, 0));
- if (!expr1)
- return NULL;
+ expr1 = ast_for_expr(c, CHILD(n, 0));
+ if (!expr1)
+ return NULL;
- expr2 = ast_for_expr(c, CHILD(n, 2));
- if (!expr2)
- return NULL;
+ expr2 = ast_for_expr(c, CHILD(n, 2));
+ if (!expr2)
+ return NULL;
- newoperator = get_operator(CHILD(n, 1));
- if (!newoperator)
- return NULL;
+ newoperator = get_operator(CHILD(n, 1));
+ if (!newoperator)
+ return NULL;
- result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
- c->c_arena);
- if (!result)
- return NULL;
+ result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
+ c->c_arena);
+ if (!result)
+ return NULL;
- nops = (NCH(n) - 1) / 2;
- for (i = 1; i < nops; i++) {
- expr_ty tmp_result, tmp;
- const node* next_oper = CHILD(n, i * 2 + 1);
+ nops = (NCH(n) - 1) / 2;
+ for (i = 1; i < nops; i++) {
+ expr_ty tmp_result, tmp;
+ const node* next_oper = CHILD(n, i * 2 + 1);
- newoperator = get_operator(next_oper);
- if (!newoperator)
- return NULL;
+ newoperator = get_operator(next_oper);
+ if (!newoperator)
+ return NULL;
- tmp = ast_for_expr(c, CHILD(n, i * 2 + 2));
- if (!tmp)
- return NULL;
+ tmp = ast_for_expr(c, CHILD(n, i * 2 + 2));
+ if (!tmp)
+ return NULL;
- tmp_result = BinOp(result, newoperator, tmp,
- LINENO(next_oper), next_oper->n_col_offset,
- c->c_arena);
- if (!tmp_result)
- return NULL;
- result = tmp_result;
- }
- return result;
+ tmp_result = BinOp(result, newoperator, tmp,
+ LINENO(next_oper), next_oper->n_col_offset,
+ c->c_arena);
+ if (!tmp_result)
+ return NULL;
+ result = tmp_result;
+ }
+ return result;
}
static expr_ty
@@ -1688,7 +1657,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
int j;
slice_ty slc;
expr_ty e;
- bool simple = true;
+ int simple = 1;
asdl_seq *slices, *elts;
slices = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!slices)
@@ -1698,7 +1667,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
if (!slc)
return NULL;
if (slc->kind != Index_kind)
- simple = false;
+ simple = 0;
asdl_seq_SET(slices, j / 2, slc);
}
if (!simple) {
@@ -1726,39 +1695,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
static expr_ty
ast_for_factor(struct compiling *c, const node *n)
{
- node *pfactor, *ppower, *patom, *pnum;
expr_ty expression;
- /* If the unary - operator is applied to a constant, don't generate
- a UNARY_NEGATIVE opcode. Just store the approriate value as a
- constant. The peephole optimizer already does something like
- this but it doesn't handle the case where the constant is
- (sys.maxint - 1). In that case, we want a PyIntObject, not a
- PyLongObject.
- */
- if (TYPE(CHILD(n, 0)) == MINUS &&
- NCH(n) == 2 &&
- TYPE((pfactor = CHILD(n, 1))) == factor &&
- NCH(pfactor) == 1 &&
- TYPE((ppower = CHILD(pfactor, 0))) == power &&
- NCH(ppower) == 1 &&
- TYPE((patom = CHILD(ppower, 0))) == atom &&
- TYPE((pnum = CHILD(patom, 0))) == NUMBER) {
- PyObject *pynum;
- char *s = PyObject_MALLOC(strlen(STR(pnum)) + 2);
- if (s == NULL)
- return NULL;
- s[0] = '-';
- strcpy(s + 1, STR(pnum));
- pynum = parsenumber(c, s);
- PyObject_FREE(s);
- if (!pynum)
- return NULL;
-
- PyArena_AddPyObject(c->c_arena, pynum);
- return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
- }
-
expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
@@ -1815,6 +1753,21 @@ ast_for_power(struct compiling *c, const node *n)
return e;
}
+static expr_ty
+ast_for_starred(struct compiling *c, const node *n)
+{
+ expr_ty tmp;
+ REQ(n, star_expr);
+
+ tmp = ast_for_expr(c, CHILD(n, 1));
+ if (!tmp)
+ return NULL;
+
+ /* The Load context is changed later. */
+ return Starred(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
+}
+
+
/* Do not name a variable 'expr'! Will cause a compile error.
*/
@@ -1823,6 +1776,7 @@ ast_for_expr(struct compiling *c, const node *n)
{
/* handle the full range of simple expressions
test: or_test ['if' or_test 'else' test] | lambdef
+ test_nocond: or_test | lambdef_nocond
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
@@ -1835,15 +1789,6 @@ ast_for_expr(struct compiling *c, const node *n)
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ('**' factor)*
-
- As well as modified versions that exist for backward compatibility,
- to explicitly allow:
- [ x for x in lambda: 0, lambda: 1 ]
- (which would be ambiguous without these extra rules)
-
- old_test: or_test | old_lambdef
- old_lambdef: 'lambda' [vararglist] ':' old_test
-
*/
asdl_seq *seq;
@@ -1852,9 +1797,9 @@ ast_for_expr(struct compiling *c, const node *n)
loop:
switch (TYPE(n)) {
case test:
- case old_test:
+ case test_nocond:
if (TYPE(CHILD(n, 0)) == lambdef ||
- TYPE(CHILD(n, 0)) == old_lambdef)
+ TYPE(CHILD(n, 0)) == lambdef_nocond)
return ast_for_lambdef(c, CHILD(n, 0));
else if (NCH(n) > 1)
return ast_for_ifexpr(c, n);
@@ -1934,6 +1879,8 @@ ast_for_expr(struct compiling *c, const node *n)
}
break;
+ case star_expr:
+ return ast_for_starred(c, n);
/* The next five cases all handle BinOps. The main body of code
is the same in each case, but the switch turned inside out to
reuse the code for each type of operator.
@@ -2051,9 +1998,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
}
else {
keyword_ty kw;
- identifier key;
+ identifier key, tmp;
int k;
- char *tmp;
/* CHILD(ch, 0) is test, but must be an identifier? */
e = ast_for_expr(c, CHILD(ch, 0));
@@ -2065,20 +2011,18 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
* then is very confusing.
*/
if (e->kind == Lambda_kind) {
- ast_error(CHILD(ch, 0),
- "lambda cannot contain assignment");
- return NULL;
+ ast_error(CHILD(ch, 0), "lambda cannot contain assignment");
+ return NULL;
} else if (e->kind != Name_kind) {
- ast_error(CHILD(ch, 0), "keyword can't be an expression");
- return NULL;
+ ast_error(CHILD(ch, 0), "keyword can't be an expression");
+ return NULL;
+ } else if (forbidden_name(e->v.Name.id, ch, 1)) {
+ return NULL;
}
key = e->v.Name.id;
- if (!forbidden_check(c, CHILD(ch, 0), PyBytes_AS_STRING(key)))
- return NULL;
for (k = 0; k < nkeywords; k++) {
- tmp = PyString_AS_STRING(
- ((keyword_ty)asdl_seq_GET(keywords, k))->arg);
- if (!strcmp(tmp, PyString_AS_STRING(key))) {
+ tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg;
+ if (!PyUnicode_Compare(tmp, key)) {
ast_error(CHILD(ch, 0), "keyword argument repeated");
return NULL;
}
@@ -2106,17 +2050,14 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
}
}
- return Call(func, args, keywords, vararg, kwarg, func->lineno,
- func->col_offset, c->c_arena);
+ return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena);
}
static expr_ty
ast_for_testlist(struct compiling *c, const node* n)
{
- /* testlist_comp: test (',' test)* [','] */
+ /* testlist_comp: test (comp_for | (',' test)* [',']) */
/* testlist: test (',' test)* [','] */
- /* testlist_safe: test (',' test)+ [','] */
- /* testlist1: test (',' test)* */
assert(NCH(n) > 0);
if (TYPE(n) == testlist_comp) {
if (NCH(n) > 1)
@@ -2124,8 +2065,7 @@ ast_for_testlist(struct compiling *c, const node* n)
}
else {
assert(TYPE(n) == testlist ||
- TYPE(n) == testlist_safe ||
- TYPE(n) == testlist1);
+ TYPE(n) == testlist_star_expr);
}
if (NCH(n) == 1)
return ast_for_expr(c, CHILD(n, 0));
@@ -2137,46 +2077,13 @@ ast_for_testlist(struct compiling *c, const node* n)
}
}
-static expr_ty
-ast_for_testlist_comp(struct compiling *c, const node* n)
-{
- /* testlist_comp: test ( comp_for | (',' test)* [','] ) */
- /* argument: test [ comp_for ] */
- assert(TYPE(n) == testlist_comp || TYPE(n) == argument);
- if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == comp_for)
- return ast_for_genexp(c, n);
- return ast_for_testlist(c, n);
-}
-
-/* like ast_for_testlist() but returns a sequence */
-static asdl_seq*
-ast_for_class_bases(struct compiling *c, const node* n)
-{
- /* testlist: test (',' test)* [','] */
- assert(NCH(n) > 0);
- REQ(n, testlist);
- if (NCH(n) == 1) {
- expr_ty base;
- asdl_seq *bases = asdl_seq_new(1, c->c_arena);
- if (!bases)
- return NULL;
- base = ast_for_expr(c, CHILD(n, 0));
- if (!base)
- return NULL;
- asdl_seq_SET(bases, 0, base);
- return bases;
- }
-
- return seq_for_testlist(c, n);
-}
-
static stmt_ty
ast_for_expr_stmt(struct compiling *c, const node *n)
{
REQ(n, expr_stmt);
- /* expr_stmt: testlist (augassign (yield_expr|testlist)
+ /* expr_stmt: testlist_star_expr (augassign (yield_expr|testlist)
| ('=' (yield_expr|testlist))*)
- testlist: test (',' test)* [',']
+ testlist_star_expr: (test|star_expr) (',' test|star_expr)* [',']
augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
| '<<=' | '>>=' | '**=' | '//='
test: ... here starts the operator precendence dance
@@ -2225,8 +2132,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (!newoperator)
return NULL;
- return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
int i;
@@ -2248,59 +2154,25 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
}
e = ast_for_testlist(c, ch);
if (!e)
- return NULL;
+ return NULL;
/* set context to assign */
if (!set_context(c, e, Store, CHILD(n, i)))
- return NULL;
+ return NULL;
asdl_seq_SET(targets, i / 2, e);
}
value = CHILD(n, NCH(n) - 1);
- if (TYPE(value) == testlist)
+ if (TYPE(value) == testlist_star_expr)
expression = ast_for_testlist(c, value);
else
expression = ast_for_expr(c, value);
if (!expression)
return NULL;
- return Assign(targets, expression, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
}
-static stmt_ty
-ast_for_print_stmt(struct compiling *c, const node *n)
-{
- /* print_stmt: 'print' ( [ test (',' test)* [','] ]
- | '>>' test [ (',' test)+ [','] ] )
- */
- expr_ty dest = NULL, expression;
- asdl_seq *seq = NULL;
- bool nl;
- int i, j, values_count, start = 1;
-
- REQ(n, print_stmt);
- if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) {
- dest = ast_for_expr(c, CHILD(n, 2));
- if (!dest)
- return NULL;
- start = 4;
- }
- values_count = (NCH(n) + 1 - start) / 2;
- if (values_count) {
- seq = asdl_seq_new(values_count, c->c_arena);
- if (!seq)
- return NULL;
- for (i = start, j = 0; i < NCH(n); i += 2, ++j) {
- expression = ast_for_expr(c, CHILD(n, i));
- if (!expression)
- return NULL;
- asdl_seq_SET(seq, j, expression);
- }
- }
- nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true;
- return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena);
-}
static asdl_seq *
ast_for_exprlist(struct compiling *c, const node *n, expr_context_ty context)
@@ -2374,48 +2246,22 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
if (!expression)
return NULL;
- return Return(expression, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
}
case raise_stmt:
if (NCH(ch) == 1)
- return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset,
- c->c_arena);
- else if (NCH(ch) == 2) {
+ return Raise(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ else if (NCH(ch) >= 2) {
+ expr_ty cause = NULL;
expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression)
return NULL;
- return Raise(expression, NULL, NULL, LINENO(n),
- n->n_col_offset, c->c_arena);
- }
- else if (NCH(ch) == 4) {
- expr_ty expr1, expr2;
-
- expr1 = ast_for_expr(c, CHILD(ch, 1));
- if (!expr1)
- return NULL;
- expr2 = ast_for_expr(c, CHILD(ch, 3));
- if (!expr2)
- return NULL;
-
- return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset,
- c->c_arena);
- }
- else if (NCH(ch) == 6) {
- expr_ty expr1, expr2, expr3;
-
- expr1 = ast_for_expr(c, CHILD(ch, 1));
- if (!expr1)
- return NULL;
- expr2 = ast_for_expr(c, CHILD(ch, 3));
- if (!expr2)
- return NULL;
- expr3 = ast_for_expr(c, CHILD(ch, 5));
- if (!expr3)
- return NULL;
-
- return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset,
- c->c_arena);
+ if (NCH(ch) == 4) {
+ cause = ast_for_expr(c, CHILD(ch, 3));
+ if (!cause)
+ return NULL;
+ }
+ return Raise(expression, cause, LINENO(n), n->n_col_offset, c->c_arena);
}
default:
PyErr_Format(PyExc_SystemError,
@@ -2435,28 +2281,28 @@ alias_for_import_name(struct compiling *c, const node *n, int store)
dotted_as_name: dotted_name ['as' NAME]
dotted_name: NAME ('.' NAME)*
*/
- PyObject *str, *name;
+ identifier str, name;
loop:
switch (TYPE(n)) {
- case import_as_name: {
+ case import_as_name: {
node *name_node = CHILD(n, 0);
str = NULL;
+ name = NEW_IDENTIFIER(name_node);
+ if (!name)
+ return NULL;
if (NCH(n) == 3) {
node *str_node = CHILD(n, 2);
- if (store && !forbidden_check(c, str_node, STR(str_node)))
- return NULL;
str = NEW_IDENTIFIER(str_node);
if (!str)
return NULL;
+ if (store && forbidden_name(str, str_node, 0))
+ return NULL;
}
else {
- if (!forbidden_check(c, name_node, STR(name_node)))
+ if (forbidden_name(name, name_node, 0))
return NULL;
}
- name = NEW_IDENTIFIER(name_node);
- if (!name)
- return NULL;
return alias(name, str, c->c_arena);
}
case dotted_as_name:
@@ -2470,22 +2316,22 @@ alias_for_import_name(struct compiling *c, const node *n, int store)
if (!a)
return NULL;
assert(!a->asname);
- if (!forbidden_check(c, asname_node, STR(asname_node)))
- return NULL;
a->asname = NEW_IDENTIFIER(asname_node);
if (!a->asname)
return NULL;
+ if (forbidden_name(a->asname, asname_node, 0))
+ return NULL;
return a;
}
break;
case dotted_name:
if (NCH(n) == 1) {
node *name_node = CHILD(n, 0);
- if (store && !forbidden_check(c, name_node, STR(name_node)))
- return NULL;
name = NEW_IDENTIFIER(name_node);
if (!name)
return NULL;
+ if (store && forbidden_name(name, name_node, 0))
+ return NULL;
return alias(name, NULL, c->c_arena);
}
else {
@@ -2493,16 +2339,17 @@ alias_for_import_name(struct compiling *c, const node *n, int store)
int i;
size_t len;
char *s;
+ PyObject *uni;
len = 0;
for (i = 0; i < NCH(n); i += 2)
/* length of string plus one for the dot */
len += strlen(STR(CHILD(n, i))) + 1;
len--; /* the last name doesn't have a dot */
- str = PyString_FromStringAndSize(NULL, len);
+ str = PyBytes_FromStringAndSize(NULL, len);
if (!str)
return NULL;
- s = PyString_AS_STRING(str);
+ s = PyBytes_AS_STRING(str);
if (!s)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
@@ -2513,13 +2360,20 @@ alias_for_import_name(struct compiling *c, const node *n, int store)
}
--s;
*s = '\0';
- PyString_InternInPlace(&str);
+ uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str),
+ PyBytes_GET_SIZE(str),
+ NULL);
+ Py_DECREF(str);
+ if (!uni)
+ return NULL;
+ str = uni;
+ PyUnicode_InternInPlace(&str);
PyArena_AddPyObject(c->c_arena, str);
return alias(str, NULL, c->c_arena);
}
break;
case STAR:
- str = PyString_InternFromString("*");
+ str = PyUnicode_InternFromString("*");
PyArena_AddPyObject(c->c_arena, str);
return alias(str, NULL, c->c_arena);
default:
@@ -2538,8 +2392,8 @@ ast_for_import_stmt(struct compiling *c, const node *n)
/*
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
- import_from: 'from' ('.'* dotted_name | '.') 'import'
- ('*' | '(' import_as_names ')' | import_as_names)
+ import_from: 'from' (('.' | '...')* dotted_name | ('.' | '...')+)
+ 'import' ('*' | '(' import_as_names ')' | import_as_names)
*/
int lineno;
int col_offset;
@@ -2555,7 +2409,7 @@ ast_for_import_stmt(struct compiling *c, const node *n)
REQ(n, dotted_as_names);
aliases = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!aliases)
- return NULL;
+ return NULL;
for (i = 0; i < NCH(n); i += 2) {
alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1);
if (!import_alias)
@@ -2579,6 +2433,10 @@ ast_for_import_stmt(struct compiling *c, const node *n)
return NULL;
idx++;
break;
+ } else if (TYPE(CHILD(n, idx)) == ELLIPSIS) {
+ /* three consecutive dots are tokenized as one ELLIPSIS */
+ ndots += 3;
+ continue;
} else if (TYPE(CHILD(n, idx)) != DOT) {
break;
}
@@ -2663,35 +2521,24 @@ ast_for_global_stmt(struct compiling *c, const node *n)
}
static stmt_ty
-ast_for_exec_stmt(struct compiling *c, const node *n)
+ast_for_nonlocal_stmt(struct compiling *c, const node *n)
{
- expr_ty expr1, globals = NULL, locals = NULL;
- int n_children = NCH(n);
- if (n_children != 2 && n_children != 4 && n_children != 6) {
- PyErr_Format(PyExc_SystemError,
- "poorly formed 'exec' statement: %d parts to statement",
- n_children);
- return NULL;
- }
+ /* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* */
+ identifier name;
+ asdl_seq *s;
+ int i;
- /* exec_stmt: 'exec' expr ['in' test [',' test]] */
- REQ(n, exec_stmt);
- expr1 = ast_for_expr(c, CHILD(n, 1));
- if (!expr1)
+ REQ(n, nonlocal_stmt);
+ s = asdl_seq_new(NCH(n) / 2, c->c_arena);
+ if (!s)
return NULL;
- if (n_children >= 4) {
- globals = ast_for_expr(c, CHILD(n, 3));
- if (!globals)
- return NULL;
- }
- if (n_children == 6) {
- locals = ast_for_expr(c, CHILD(n, 5));
- if (!locals)
+ for (i = 1; i < NCH(n); i += 2) {
+ name = NEW_IDENTIFIER(CHILD(n, i));
+ if (!name)
return NULL;
+ asdl_seq_SET(s, i / 2, name);
}
-
- return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
@@ -2703,8 +2550,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
expr_ty expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
- return Assert(expression, NULL, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 4) {
expr_ty expr1, expr2;
@@ -2926,8 +2772,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
- return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 7) {
expr_ty expression;
@@ -2943,8 +2788,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
if (!seq2)
return NULL;
- return While(expression, seq1, seq2, LINENO(n), n->n_col_offset,
- c->c_arena);
+ return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@@ -2995,7 +2839,7 @@ ast_for_for_stmt(struct compiling *c, const node *n)
static excepthandler_ty
ast_for_except_clause(struct compiling *c, const node *exc, node *body)
{
- /* except_clause: 'except' [test [(',' | 'as') test]] */
+ /* except_clause: 'except' [test ['as' test]] */
REQ(exc, except_clause);
REQ(body, suite);
@@ -3024,10 +2868,10 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
else if (NCH(exc) == 4) {
asdl_seq *suite_seq;
expr_ty expression;
- expr_ty e = ast_for_expr(c, CHILD(exc, 3));
+ identifier e = NEW_IDENTIFIER(CHILD(exc, 3));
if (!e)
return NULL;
- if (!set_context(c, e, Store, CHILD(exc, 3)))
+ if (forbidden_name(e, CHILD(exc, 3), 0))
return NULL;
expression = ast_for_expr(c, CHILD(exc, 1));
if (!expression)
@@ -3187,50 +3031,64 @@ ast_for_with_stmt(struct compiling *c, const node *n)
static stmt_ty
ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
- /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */
+ /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */
PyObject *classname;
- asdl_seq *bases, *s;
+ asdl_seq *s;
+ expr_ty call;
REQ(n, classdef);
- if (!forbidden_check(c, n, STR(CHILD(n, 1))))
- return NULL;
-
- if (NCH(n) == 4) {
+ if (NCH(n) == 4) { /* class NAME ':' suite */
s = ast_for_suite(c, CHILD(n, 3));
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
- return ClassDef(classname, NULL, s, decorator_seq, LINENO(n),
- n->n_col_offset, c->c_arena);
+ if (forbidden_name(classname, CHILD(n, 3), 0))
+ return NULL;
+ return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
+ LINENO(n), n->n_col_offset, c->c_arena);
}
- /* check for empty base list */
- if (TYPE(CHILD(n,3)) == RPAR) {
+
+ if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
s = ast_for_suite(c, CHILD(n,5));
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
- return ClassDef(classname, NULL, s, decorator_seq, LINENO(n),
- n->n_col_offset, c->c_arena);
+ if (forbidden_name(classname, CHILD(n, 3), 0))
+ return NULL;
+ return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
+ LINENO(n), n->n_col_offset, c->c_arena);
}
- /* else handle the base class list */
- bases = ast_for_class_bases(c, CHILD(n, 3));
- if (!bases)
- return NULL;
-
+ /* class NAME '(' arglist ')' ':' suite */
+ /* build up a fake Call node so we can extract its pieces */
+ {
+ PyObject *dummy_name;
+ expr_ty dummy;
+ dummy_name = NEW_IDENTIFIER(CHILD(n, 1));
+ if (!dummy_name)
+ return NULL;
+ dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ call = ast_for_call(c, CHILD(n, 3), dummy);
+ if (!call)
+ return NULL;
+ }
s = ast_for_suite(c, CHILD(n, 6));
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
- return ClassDef(classname, bases, s, decorator_seq,
- LINENO(n), n->n_col_offset, c->c_arena);
+ if (forbidden_name(classname, CHILD(n, 1), 0))
+ return NULL;
+
+ return ClassDef(classname, call->v.Call.args, call->v.Call.keywords,
+ call->v.Call.starargs, call->v.Call.kwargs, s,
+ decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
@@ -3246,15 +3104,12 @@ ast_for_stmt(struct compiling *c, const node *n)
}
if (TYPE(n) == small_stmt) {
n = CHILD(n, 0);
- /* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt
- | flow_stmt | import_stmt | global_stmt | exec_stmt
- | assert_stmt
+ /* small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt
+ | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
*/
switch (TYPE(n)) {
case expr_stmt:
return ast_for_expr_stmt(c, n);
- case print_stmt:
- return ast_for_print_stmt(c, n);
case del_stmt:
return ast_for_del_stmt(c, n);
case pass_stmt:
@@ -3265,8 +3120,8 @@ ast_for_stmt(struct compiling *c, const node *n)
return ast_for_import_stmt(c, n);
case global_stmt:
return ast_for_global_stmt(c, n);
- case exec_stmt:
- return ast_for_exec_stmt(c, n);
+ case nonlocal_stmt:
+ return ast_for_nonlocal_stmt(c, n);
case assert_stmt:
return ast_for_assert_stmt(c, n);
default:
@@ -3311,253 +3166,256 @@ ast_for_stmt(struct compiling *c, const node *n)
static PyObject *
parsenumber(struct compiling *c, const char *s)
{
- const char *end;
- long x;
- double dx;
-#ifndef WITHOUT_COMPLEX
- Py_complex complex;
- int imflag;
-#endif
-
- assert(s != NULL);
- errno = 0;
- end = s + strlen(s) - 1;
-#ifndef WITHOUT_COMPLEX
- imflag = *end == 'j' || *end == 'J';
-#endif
- if (*end == 'l' || *end == 'L')
- return PyLong_FromString((char *)s, (char **)0, 0);
- x = PyOS_strtol((char *)s, (char **)&end, 0);
- if (*end == '\0') {
- if (errno != 0)
- return PyLong_FromString((char *)s, (char **)0, 0);
- return PyInt_FromLong(x);
- }
- /* XXX Huge floats may silently fail */
-#ifndef WITHOUT_COMPLEX
- if (imflag) {
- complex.real = 0.;
- complex.imag = PyOS_string_to_double(s, (char **)&end, NULL);
- if (complex.imag == -1.0 && PyErr_Occurred())
- return NULL;
- return PyComplex_FromCComplex(complex);
- }
- else
-#endif
- {
- dx = PyOS_string_to_double(s, NULL, NULL);
- if (dx == -1.0 && PyErr_Occurred())
- return NULL;
- return PyFloat_FromDouble(dx);
+ const char *end;
+ long x;
+ double dx;
+ Py_complex compl;
+ int imflag;
+
+ assert(s != NULL);
+ errno = 0;
+ end = s + strlen(s) - 1;
+ imflag = *end == 'j' || *end == 'J';
+ if (s[0] == '0') {
+ x = (long) PyOS_strtoul((char *)s, (char **)&end, 0);
+ if (x < 0 && errno == 0) {
+ return PyLong_FromString((char *)s,
+ (char **)0,
+ 0);
}
+ }
+ else
+ x = PyOS_strtol((char *)s, (char **)&end, 0);
+ if (*end == '\0') {
+ if (errno != 0)
+ return PyLong_FromString((char *)s, (char **)0, 0);
+ return PyLong_FromLong(x);
+ }
+ /* XXX Huge floats may silently fail */
+ if (imflag) {
+ compl.real = 0.;
+ compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
+ if (compl.imag == -1.0 && PyErr_Occurred())
+ return NULL;
+ return PyComplex_FromCComplex(compl);
+ }
+ else
+ {
+ dx = PyOS_string_to_double(s, NULL, NULL);
+ if (dx == -1.0 && PyErr_Occurred())
+ return NULL;
+ return PyFloat_FromDouble(dx);
+ }
}
static PyObject *
decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encoding)
{
-#ifndef Py_USING_UNICODE
- Py_FatalError("decode_utf8 should not be called in this build.");
+ PyObject *u, *v;
+ char *s, *t;
+ t = s = (char *)*sPtr;
+ /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
+ while (s < end && (*s & 0x80)) s++;
+ *sPtr = s;
+ u = PyUnicode_DecodeUTF8(t, s - t, NULL);
+ if (u == NULL)
return NULL;
-#else
- PyObject *u, *v;
- char *s, *t;
- t = s = (char *)*sPtr;
- /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
- while (s < end && (*s & 0x80)) s++;
- *sPtr = s;
- u = PyUnicode_DecodeUTF8(t, s - t, NULL);
- if (u == NULL)
- return NULL;
- v = PyUnicode_AsEncodedString(u, encoding, NULL);
- Py_DECREF(u);
- return v;
-#endif
+ v = PyUnicode_AsEncodedString(u, encoding, NULL);
+ Py_DECREF(u);
+ return v;
}
-#ifdef Py_USING_UNICODE
static PyObject *
decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding)
{
- PyObject *v;
- PyObject *u = NULL;
- char *buf;
- char *p;
- const char *end;
- if (encoding != NULL && strcmp(encoding, "iso-8859-1")) {
- /* check for integer overflow */
- if (len > PY_SIZE_MAX / 6)
- return NULL;
- /* "<C3><A4>" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
- "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
- u = PyString_FromStringAndSize((char *)NULL, len * 6);
- if (u == NULL)
- return NULL;
- p = buf = PyString_AsString(u);
- end = s + len;
- while (s < end) {
- if (*s == '\\') {
- *p++ = *s++;
- if (*s & 0x80) {
- strcpy(p, "u005c");
- p += 5;
- }
- }
- if (*s & 0x80) { /* XXX inefficient */
- PyObject *w;
- char *r;
- Py_ssize_t rn, i;
- w = decode_utf8(c, &s, end, "utf-32-be");
- if (w == NULL) {
- Py_DECREF(u);
- return NULL;
- }
- r = PyString_AsString(w);
- rn = PyString_Size(w);
- assert(rn % 4 == 0);
- for (i = 0; i < rn; i += 4) {
- sprintf(p, "\\U%02x%02x%02x%02x",
- r[i + 0] & 0xFF,
- r[i + 1] & 0xFF,
- r[i + 2] & 0xFF,
- r[i + 3] & 0xFF);
- p += 10;
- }
- Py_DECREF(w);
- } else {
- *p++ = *s++;
- }
+ PyObject *v, *u;
+ char *buf;
+ char *p;
+ const char *end;
+
+ if (encoding == NULL) {
+ buf = (char *)s;
+ u = NULL;
+ } else {
+ /* check for integer overflow */
+ if (len > PY_SIZE_MAX / 6)
+ return NULL;
+ /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
+ "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
+ u = PyBytes_FromStringAndSize((char *)NULL, len * 6);
+ if (u == NULL)
+ return NULL;
+ p = buf = PyBytes_AsString(u);
+ end = s + len;
+ while (s < end) {
+ if (*s == '\\') {
+ *p++ = *s++;
+ if (*s & 0x80) {
+ strcpy(p, "u005c");
+ p += 5;
+ }
+ }
+ if (*s & 0x80) { /* XXX inefficient */
+ PyObject *w;
+ char *r;
+ Py_ssize_t rn, i;
+ w = decode_utf8(c, &s, end, "utf-32-be");
+ if (w == NULL) {
+ Py_DECREF(u);
+ return NULL;
+ }
+ r = PyBytes_AS_STRING(w);
+ rn = Py_SIZE(w);
+ assert(rn % 4 == 0);
+ for (i = 0; i < rn; i += 4) {
+ sprintf(p, "\\U%02x%02x%02x%02x",
+ r[i + 0] & 0xFF,
+ r[i + 1] & 0xFF,
+ r[i + 2] & 0xFF,
+ r[i + 3] & 0xFF);
+ p += 10;
}
- len = p - buf;
- s = buf;
+ /* Should be impossible to overflow */
+ assert(p - buf <= Py_SIZE(u));
+ Py_DECREF(w);
+ } else {
+ *p++ = *s++;
+ }
}
- if (rawmode)
- v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL);
- else
- v = PyUnicode_DecodeUnicodeEscape(s, len, NULL);
- Py_XDECREF(u);
- return v;
+ len = p - buf;
+ s = buf;
+ }
+ if (rawmode)
+ v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL);
+ else
+ v = PyUnicode_DecodeUnicodeEscape(s, len, NULL);
+ Py_XDECREF(u);
+ return v;
}
-#endif
/* s is a Python string literal, including the bracketing quote characters,
- * and r &/or u prefixes (if any), and embedded escape sequences (if any).
+ * and r &/or b prefixes (if any), and embedded escape sequences (if any).
* parsestr parses it, and returns the decoded Python string object.
*/
static PyObject *
-parsestr(struct compiling *c, const char *s)
-{
- size_t len;
- int quote = Py_CHARMASK(*s);
- int rawmode = 0;
- int need_encoding;
- int unicode = c->c_future_unicode;
-
- if (isalpha(quote) || quote == '_') {
- if (quote == 'u' || quote == 'U') {
- quote = *++s;
- unicode = 1;
- }
- if (quote == 'b' || quote == 'B') {
- quote = *++s;
- unicode = 0;
- }
- if (quote == 'r' || quote == 'R') {
- quote = *++s;
- rawmode = 1;
- }
+parsestr(struct compiling *c, const node *n, int *bytesmode)
+{
+ size_t len;
+ const char *s = STR(n);
+ int quote = Py_CHARMASK(*s);
+ int rawmode = 0;
+ int need_encoding;
+ if (isalpha(quote)) {
+ if (quote == 'b' || quote == 'B') {
+ quote = *++s;
+ *bytesmode = 1;
}
- if (quote != '\'' && quote != '\"') {
- PyErr_BadInternalCall();
- return NULL;
+ if (quote == 'r' || quote == 'R') {
+ quote = *++s;
+ rawmode = 1;
}
- s++;
- len = strlen(s);
- if (len > INT_MAX) {
- PyErr_SetString(PyExc_OverflowError,
- "string to parse is too long");
- return NULL;
+ }
+ if (quote != '\'' && quote != '\"') {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ s++;
+ len = strlen(s);
+ if (len > INT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "string to parse is too long");
+ return NULL;
+ }
+ if (s[--len] != quote) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ if (len >= 4 && s[0] == quote && s[1] == quote) {
+ s += 2;
+ len -= 2;
+ if (s[--len] != quote || s[--len] != quote) {
+ PyErr_BadInternalCall();
+ return NULL;
}
- if (s[--len] != quote) {
- PyErr_BadInternalCall();
+ }
+ if (!*bytesmode && !rawmode) {
+ return decode_unicode(c, s, len, rawmode, c->c_encoding);
+ }
+ if (*bytesmode) {
+ /* Disallow non-ascii characters (but not escapes) */
+ const char *c;
+ for (c = s; *c; c++) {
+ if (Py_CHARMASK(*c) >= 0x80) {
+ ast_error(n, "bytes can only contain ASCII "
+ "literal characters.");
return NULL;
+ }
}
- if (len >= 4 && s[0] == quote && s[1] == quote) {
- s += 2;
- len -= 2;
- if (s[--len] != quote || s[--len] != quote) {
- PyErr_BadInternalCall();
- return NULL;
- }
- }
-#ifdef Py_USING_UNICODE
- if (unicode || Py_UnicodeFlag) {
- return decode_unicode(c, s, len, rawmode, c->c_encoding);
- }
-#endif
- need_encoding = (c->c_encoding != NULL &&
- strcmp(c->c_encoding, "utf-8") != 0 &&
- strcmp(c->c_encoding, "iso-8859-1") != 0);
- if (rawmode || strchr(s, '\\') == NULL) {
- if (need_encoding) {
-#ifndef Py_USING_UNICODE
- /* This should not happen - we never see any other
- encoding. */
- Py_FatalError(
- "cannot deal with encodings in this build.");
-#else
- PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
- if (u == NULL)
- return NULL;
- v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
- Py_DECREF(u);
- return v;
-#endif
- } else {
- return PyString_FromStringAndSize(s, len);
- }
+ }
+ need_encoding = (!*bytesmode && c->c_encoding != NULL &&
+ strcmp(c->c_encoding, "utf-8") != 0);
+ if (rawmode || strchr(s, '\\') == NULL) {
+ if (need_encoding) {
+ PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
+ if (u == NULL || !*bytesmode)
+ return u;
+ v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
+ Py_DECREF(u);
+ return v;
+ } else if (*bytesmode) {
+ return PyBytes_FromStringAndSize(s, len);
+ } else if (strcmp(c->c_encoding, "utf-8") == 0) {
+ return PyUnicode_FromStringAndSize(s, len);
+ } else {
+ return PyUnicode_DecodeLatin1(s, len, NULL);
}
-
- return PyString_DecodeEscape(s, len, NULL, unicode,
- need_encoding ? c->c_encoding : NULL);
+ }
+ return PyBytes_DecodeEscape(s, len, NULL, 1,
+ need_encoding ? c->c_encoding : NULL);
}
-/* Build a Python string object out of a STRING atom. This takes care of
+/* Build a Python string object out of a STRING+ atom. This takes care of
* compile-time literal catenation, calling parsestr() on each piece, and
* pasting the intermediate results together.
*/
static PyObject *
-parsestrplus(struct compiling *c, const node *n)
+parsestrplus(struct compiling *c, const node *n, int *bytesmode)
{
- PyObject *v;
- int i;
- REQ(CHILD(n, 0), STRING);
- if ((v = parsestr(c, STR(CHILD(n, 0)))) != NULL) {
- /* String literal concatenation */
- for (i = 1; i < NCH(n); i++) {
- PyObject *s;
- s = parsestr(c, STR(CHILD(n, i)));
- if (s == NULL)
- goto onError;
- if (PyString_Check(v) && PyString_Check(s)) {
- PyString_ConcatAndDel(&v, s);
- if (v == NULL)
- goto onError;
- }
-#ifdef Py_USING_UNICODE
- else {
- PyObject *temp = PyUnicode_Concat(v, s);
- Py_DECREF(s);
- Py_DECREF(v);
- v = temp;
- if (v == NULL)
- goto onError;
- }
-#endif
- }
+ PyObject *v;
+ int i;
+ REQ(CHILD(n, 0), STRING);
+ v = parsestr(c, CHILD(n, 0), bytesmode);
+ if (v != NULL) {
+ /* String literal concatenation */
+ for (i = 1; i < NCH(n); i++) {
+ PyObject *s;
+ int subbm = 0;
+ s = parsestr(c, CHILD(n, i), &subbm);
+ if (s == NULL)
+ goto onError;
+ if (*bytesmode != subbm) {
+ ast_error(n, "cannot mix bytes and nonbytes literals");
+ Py_DECREF(s);
+ goto onError;
+ }
+ if (PyBytes_Check(v) && PyBytes_Check(s)) {
+ PyBytes_ConcatAndDel(&v, s);
+ if (v == NULL)
+ goto onError;
+ }
+ else {
+ PyObject *temp = PyUnicode_Concat(v, s);
+ Py_DECREF(s);
+ Py_DECREF(v);
+ v = temp;
+ if (v == NULL)
+ goto onError;
+ }
}
- return v;
+ }
+ return v;
- onError:
- Py_XDECREF(v);
- return NULL;
+ onError:
+ Py_XDECREF(v);
+ return NULL;
}
View Lorry Controller Status