diff options
| author | Bob Weinand <bobwei9@hotmail.com> | 2015-05-25 18:38:40 +0200 |
|---|---|---|
| committer | Bob Weinand <bobwei9@hotmail.com> | 2015-05-25 18:38:40 +0200 |
| commit | 886cbea94f7f823f5b7afbbafe023a9068f069e5 (patch) | |
| tree | 7eaa8f7476aa704c69a72ad951e1afe7ae46ecd1 | |
| parent | 18cd4b17cc66df57d7806b7b574aff0f518aabf4 (diff) | |
| parent | c2f3091b987dd7e0fc7a5400dca2395c3eb25d9f (diff) | |
| download | php-git-886cbea94f7f823f5b7afbbafe023a9068f069e5.tar.gz | |
Merge context sensitive lexer RFC
36 files changed, 2045 insertions, 248 deletions
diff --git a/Zend/tests/grammar/regression_001.phpt b/Zend/tests/grammar/regression_001.phpt new file mode 100644 index 0000000000..73d5eacdf6 --- /dev/null +++ b/Zend/tests/grammar/regression_001.phpt @@ -0,0 +1,33 @@ +--TEST-- +Test to check static method calls syntax regression +--FILE-- +<?php + +class Foo { + public static function function(){ echo __METHOD__, PHP_EOL; } +} + +Foo::function(); + +Foo:: +function(); + +Foo:: + function(); + + +Foo:: + function( + +); + +echo "\nDone\n"; + +--EXPECTF-- + +Foo::function +Foo::function +Foo::function +Foo::function + +Done diff --git a/Zend/tests/grammar/regression_002.phpt b/Zend/tests/grammar/regression_002.phpt new file mode 100644 index 0000000000..dd307c99d8 --- /dev/null +++ b/Zend/tests/grammar/regression_002.phpt @@ -0,0 +1,22 @@ +--TEST-- +Test to ensure ::class still works +--FILE-- +<?php + +class Foo {} + +var_dump(Foo::class); + +var_dump(Foo:: class); + +var_dump(Foo:: CLASS); + +var_dump(Foo:: + +CLASS); + +--EXPECTF-- +string(3) "Foo" +string(3) "Foo" +string(3) "Foo" +string(3) "Foo" diff --git a/Zend/tests/grammar/regression_003.phpt b/Zend/tests/grammar/regression_003.phpt new file mode 100644 index 0000000000..7213ca3e07 --- /dev/null +++ b/Zend/tests/grammar/regression_003.phpt @@ -0,0 +1,12 @@ +--TEST-- +Test to ensure ::class is still reserved in obj scope +--FILE-- +<?php + +class Obj +{ + const CLASS = 'class'; +} + +--EXPECTF-- +Parse error: syntax error, unexpected 'CLASS' (T_CLASS) in %s on line 5 diff --git a/Zend/tests/grammar/regression_004.phpt b/Zend/tests/grammar/regression_004.phpt new file mode 100644 index 0000000000..e95674d8c9 --- /dev/null +++ b/Zend/tests/grammar/regression_004.phpt @@ -0,0 +1,15 @@ +--TEST-- +Test possible function naming regression on procedural scope +--FILE-- +<?php + +class Obj +{ + function echo(){} // valid + function return(){} // valid +} + +function echo(){} // not valid + +--EXPECTF-- +Parse error: syntax error, unexpected 'echo' (T_ECHO), expecting identifier (T_STRING) or '(' in %s on line 9 diff --git a/Zend/tests/grammar/regression_005.phpt b/Zend/tests/grammar/regression_005.phpt new file mode 100644 index 0000000000..7704375d6e --- /dev/null +++ b/Zend/tests/grammar/regression_005.phpt @@ -0,0 +1,14 @@ +--TEST-- +Test possible constant naming regression on procedural scope +--FILE-- +<?php + +class Obj +{ + const return = 'yep'; +} + +const return = 'nope'; + +--EXPECTF-- +Parse error: syntax error, unexpected 'return' (T_RETURN), expecting identifier (T_STRING) in %s on line 8 diff --git a/Zend/tests/grammar/regression_006.phpt b/Zend/tests/grammar/regression_006.phpt new file mode 100644 index 0000000000..6aae0ba24b --- /dev/null +++ b/Zend/tests/grammar/regression_006.phpt @@ -0,0 +1,30 @@ +--TEST-- +Test to ensure const list syntax declaration works +--FILE-- +<?php + +class Obj +{ + const DECLARE = 'declare', + RETURN = 'return', + FUNCTION = 'function', + USE = 'use'; +} + +echo Obj::DECLARE, PHP_EOL; +echo Obj::RETURN, PHP_EOL; +echo Obj::FUNCTION, PHP_EOL; +echo Obj::USE, PHP_EOL; +echo Obj:: + + USE, PHP_EOL; +echo "\nDone\n"; + +--EXPECTF-- +declare +return +function +use +use + +Done diff --git a/Zend/tests/grammar/regression_007.phpt b/Zend/tests/grammar/regression_007.phpt new file mode 100644 index 0000000000..92b22531a4 --- /dev/null +++ b/Zend/tests/grammar/regression_007.phpt @@ -0,0 +1,44 @@ +--TEST-- +Test to ensure semi reserved words allow deference +--FILE-- +<?php + +class Foo { + const use = 'yay'; + + public static function new() { + echo __METHOD__, PHP_EOL; + return new static(); + } + + public function self() { + echo __METHOD__, PHP_EOL; + return $this; + } +} + +Foo::new()::new()::new(); + +var_dump( + (new Foo)->self()::new()->self()->self()::use +); + +Foo::{'new'}(); + +var_dump(Foo::use); + +echo "\nDone\n"; + +--EXPECTF-- +Foo::new +Foo::new +Foo::new +Foo::self +Foo::new +Foo::self +Foo::self +string(3) "yay" +Foo::new +string(3) "yay" + +Done diff --git a/Zend/tests/grammar/regression_008.phpt b/Zend/tests/grammar/regression_008.phpt new file mode 100644 index 0000000000..7741ed036c --- /dev/null +++ b/Zend/tests/grammar/regression_008.phpt @@ -0,0 +1,21 @@ +--TEST-- +Test to check regressions on string interpolation with class members access +--FILE-- +<?php + +class Friday { + public $require = "fun"; +} + +$friday = new Friday; + +echo "$friday->require ($friday->require) {$friday->require}", PHP_EOL; + +echo "\nDone\n"; + + +--EXPECTF-- + +fun (fun) fun + +Done diff --git a/Zend/tests/grammar/regression_009.phpt b/Zend/tests/grammar/regression_009.phpt new file mode 100644 index 0000000000..589d90316b --- /dev/null +++ b/Zend/tests/grammar/regression_009.phpt @@ -0,0 +1,18 @@ +--TEST-- +Test to check regressions on use statements and lexer state +--FILE-- +<?php + +use A\B\C\D; + +class Foo +{ + private static $foo; + +} + +echo PHP_EOL, "Done", PHP_EOL; + +--EXPECTF-- + +Done diff --git a/Zend/tests/grammar/regression_010.phpt b/Zend/tests/grammar/regression_010.phpt new file mode 100644 index 0000000000..5dc90f288a --- /dev/null +++ b/Zend/tests/grammar/regression_010.phpt @@ -0,0 +1,14 @@ +--TEST-- +Test to check regressions on T_IMPLEMENTS followed by a T_NS_SEPARATOR +--FILE-- +<?php + +interface A{} + +class B implements\A {} + +echo "Done", PHP_EOL; + +--EXPECTF-- + +Done diff --git a/Zend/tests/grammar/regression_011.phpt b/Zend/tests/grammar/regression_011.phpt new file mode 100644 index 0000000000..c79c077187 --- /dev/null +++ b/Zend/tests/grammar/regression_011.phpt @@ -0,0 +1,18 @@ +--TEST-- +Testing instantiation using namespace:: prefix +--FILE-- +<?php + +namespace foo; + +class bar { +} + +class_alias('foo\bar', 'foo\baz'); + +var_dump(new namespace\baz); + +?> +--EXPECTF-- +object(foo\bar)#%d (0) { +} diff --git a/Zend/tests/grammar/regression_012.phpt b/Zend/tests/grammar/regression_012.phpt new file mode 100644 index 0000000000..3b4925afa6 --- /dev/null +++ b/Zend/tests/grammar/regression_012.phpt @@ -0,0 +1,13 @@ +--TEST-- +Testing for regression on const list syntax and arrays +--FILE-- +<?php + +class A { + const A = [1, FOREACH]; +} + +?> +--EXPECTF-- + +Parse error: syntax error, unexpected 'FOREACH' (T_FOREACH), expecting ']' in %s on line %d diff --git a/Zend/tests/grammar/regression_013.phpt b/Zend/tests/grammar/regression_013.phpt new file mode 100644 index 0000000000..1c60ffc273 --- /dev/null +++ b/Zend/tests/grammar/regression_013.phpt @@ -0,0 +1,13 @@ +--TEST-- +Testing for regression with encapsed variables in class declaration context +--FILE-- +<?php + +class A { function foo() { "{${$a}}"; } function list() {} } + +echo "Done", PHP_EOL; + +?> +--EXPECTF-- + +Done diff --git a/Zend/tests/grammar/semi_reserved_001.phpt b/Zend/tests/grammar/semi_reserved_001.phpt new file mode 100644 index 0000000000..06b2532fb6 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_001.phpt @@ -0,0 +1,188 @@ +--TEST-- +Test semi-reserved words as class methods +--FILE-- +<?php + +class Obj +{ + function empty(){ echo __METHOD__, PHP_EOL; } + function callable(){ echo __METHOD__, PHP_EOL; } + function trait(){ echo __METHOD__, PHP_EOL; } + function extends(){ echo __METHOD__, PHP_EOL; } + function implements(){ echo __METHOD__, PHP_EOL; } + function const(){ echo __METHOD__, PHP_EOL; } + function enddeclare(){ echo __METHOD__, PHP_EOL; } + function endfor(){ echo __METHOD__, PHP_EOL; } + function endforeach(){ echo __METHOD__, PHP_EOL; } + function endif(){ echo __METHOD__, PHP_EOL; } + function endwhile(){ echo __METHOD__, PHP_EOL; } + function and(){ echo __METHOD__, PHP_EOL; } + function global(){ echo __METHOD__, PHP_EOL; } + function goto(){ echo __METHOD__, PHP_EOL; } + function instanceof(){ echo __METHOD__, PHP_EOL; } + function insteadof(){ echo __METHOD__, PHP_EOL; } + function interface(){ echo __METHOD__, PHP_EOL; } + function new(){ echo __METHOD__, PHP_EOL; } + function or(){ echo __METHOD__, PHP_EOL; } + function xor(){ echo __METHOD__, PHP_EOL; } + function try(){ echo __METHOD__, PHP_EOL; } + function use(){ echo __METHOD__, PHP_EOL; } + function var(){ echo __METHOD__, PHP_EOL; } + function exit(){ echo __METHOD__, PHP_EOL; } + function list(){ echo __METHOD__, PHP_EOL; } + function clone(){ echo __METHOD__, PHP_EOL; } + function include(){ echo __METHOD__, PHP_EOL; } + function include_once(){ echo __METHOD__, PHP_EOL; } + function throw(){ echo __METHOD__, PHP_EOL; } + function array(){ echo __METHOD__, PHP_EOL; } + function print(){ echo __METHOD__, PHP_EOL; } + function echo(){ echo __METHOD__, PHP_EOL; } + function require(){ echo __METHOD__, PHP_EOL; } + function require_once(){ echo __METHOD__, PHP_EOL; } + function return(){ echo __METHOD__, PHP_EOL; } + function else(){ echo __METHOD__, PHP_EOL; } + function elseif(){ echo __METHOD__, PHP_EOL; } + function default(){ echo __METHOD__, PHP_EOL; } + function break(){ echo __METHOD__, PHP_EOL; } + function continue(){ echo __METHOD__, PHP_EOL; } + function switch(){ echo __METHOD__, PHP_EOL; } + function yield(){ echo __METHOD__, PHP_EOL; } + function function(){ echo __METHOD__, PHP_EOL; } + function if(){ echo __METHOD__, PHP_EOL; } + function endswitch(){ echo __METHOD__, PHP_EOL; } + function finally(){ echo __METHOD__, PHP_EOL; } + function for(){ echo __METHOD__, PHP_EOL; } + function foreach(){ echo __METHOD__, PHP_EOL; } + function declare(){ echo __METHOD__, PHP_EOL; } + function case(){ echo __METHOD__, PHP_EOL; } + function do(){ echo __METHOD__, PHP_EOL; } + function while(){ echo __METHOD__, PHP_EOL; } + function as(){ echo __METHOD__, PHP_EOL; } + function catch(){ echo __METHOD__, PHP_EOL; } + function die(){ echo __METHOD__, PHP_EOL; } + function self(){ echo __METHOD__, PHP_EOL; } + function parent(){ echo __METHOD__, PHP_EOL; } +} + +$obj = new Obj; + +$obj->empty(); +$obj->callable(); +$obj->trait(); +$obj->extends(); +$obj->implements(); +$obj->const(); +$obj->enddeclare(); +$obj->endfor(); +$obj->endforeach(); +$obj->endif(); +$obj->endwhile(); +$obj->and(); +$obj->global(); +$obj->goto(); +$obj->instanceof(); +$obj->insteadof(); +$obj->interface(); +$obj->new(); +$obj->or(); +$obj->xor(); +$obj->try(); +$obj->use(); +$obj->var(); +$obj->exit(); +$obj->list(); +$obj->clone(); +$obj->include(); +$obj->include_once(); +$obj->throw(); +$obj->array(); +$obj->print(); +$obj->echo(); +$obj->require(); +$obj->require_once(); +$obj->return(); +$obj->else(); +$obj->elseif(); +$obj->default(); +$obj->break(); +$obj->continue(); +$obj->switch(); +$obj->yield(); +$obj->function(); +$obj->if(); +$obj->endswitch(); +$obj->finally(); +$obj->for(); +$obj->foreach(); +$obj->declare(); +$obj->case(); +$obj->do(); +$obj->while(); +$obj->as(); +$obj->catch(); +$obj->die(); +$obj->self(); +$obj->parent(); + +echo "\nDone\n"; + +--EXPECTF-- +Obj::empty +Obj::callable +Obj::trait +Obj::extends +Obj::implements +Obj::const +Obj::enddeclare +Obj::endfor +Obj::endforeach +Obj::endif +Obj::endwhile +Obj::and +Obj::global +Obj::goto +Obj::instanceof +Obj::insteadof +Obj::interface +Obj::new +Obj::or +Obj::xor +Obj::try +Obj::use +Obj::var +Obj::exit +Obj::list +Obj::clone +Obj::include +Obj::include_once +Obj::throw +Obj::array +Obj::print +Obj::echo +Obj::require +Obj::require_once +Obj::return +Obj::else +Obj::elseif +Obj::default +Obj::break +Obj::continue +Obj::switch +Obj::yield +Obj::function +Obj::if +Obj::endswitch +Obj::finally +Obj::for +Obj::foreach +Obj::declare +Obj::case +Obj::do +Obj::while +Obj::as +Obj::catch +Obj::die +Obj::self +Obj::parent + +Done diff --git a/Zend/tests/grammar/semi_reserved_002.phpt b/Zend/tests/grammar/semi_reserved_002.phpt new file mode 100644 index 0000000000..e4c49cd7a8 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_002.phpt @@ -0,0 +1,186 @@ +--TEST-- +Test semi-reserved words as static class methods +--FILE-- +<?php + +class Obj +{ + static function empty(){ echo __METHOD__, PHP_EOL; } + static function callable(){ echo __METHOD__, PHP_EOL; } + static function trait(){ echo __METHOD__, PHP_EOL; } + static function extends(){ echo __METHOD__, PHP_EOL; } + static function implements(){ echo __METHOD__, PHP_EOL; } + static function const(){ echo __METHOD__, PHP_EOL; } + static function enddeclare(){ echo __METHOD__, PHP_EOL; } + static function endfor(){ echo __METHOD__, PHP_EOL; } + static function endforeach(){ echo __METHOD__, PHP_EOL; } + static function endif(){ echo __METHOD__, PHP_EOL; } + static function endwhile(){ echo __METHOD__, PHP_EOL; } + static function and(){ echo __METHOD__, PHP_EOL; } + static function global(){ echo __METHOD__, PHP_EOL; } + static function goto(){ echo __METHOD__, PHP_EOL; } + static function instanceof(){ echo __METHOD__, PHP_EOL; } + static function insteadof(){ echo __METHOD__, PHP_EOL; } + static function interface(){ echo __METHOD__, PHP_EOL; } + static function new(){ echo __METHOD__, PHP_EOL; } + static function or(){ echo __METHOD__, PHP_EOL; } + static function xor(){ echo __METHOD__, PHP_EOL; } + static function try(){ echo __METHOD__, PHP_EOL; } + static function use(){ echo __METHOD__, PHP_EOL; } + static function var(){ echo __METHOD__, PHP_EOL; } + static function exit(){ echo __METHOD__, PHP_EOL; } + static function list(){ echo __METHOD__, PHP_EOL; } + static function clone(){ echo __METHOD__, PHP_EOL; } + static function include(){ echo __METHOD__, PHP_EOL; } + static function include_once(){ echo __METHOD__, PHP_EOL; } + static function throw(){ echo __METHOD__, PHP_EOL; } + static function array(){ echo __METHOD__, PHP_EOL; } + static function print(){ echo __METHOD__, PHP_EOL; } + static function echo(){ echo __METHOD__, PHP_EOL; } + static function require(){ echo __METHOD__, PHP_EOL; } + static function require_once(){ echo __METHOD__, PHP_EOL; } + static function return(){ echo __METHOD__, PHP_EOL; } + static function else(){ echo __METHOD__, PHP_EOL; } + static function elseif(){ echo __METHOD__, PHP_EOL; } + static function default(){ echo __METHOD__, PHP_EOL; } + static function break(){ echo __METHOD__, PHP_EOL; } + static function continue(){ echo __METHOD__, PHP_EOL; } + static function switch(){ echo __METHOD__, PHP_EOL; } + static function yield(){ echo __METHOD__, PHP_EOL; } + static function function(){ echo __METHOD__, PHP_EOL; } + static function if(){ echo __METHOD__, PHP_EOL; } + static function endswitch(){ echo __METHOD__, PHP_EOL; } + static function finally(){ echo __METHOD__, PHP_EOL; } + static function for(){ echo __METHOD__, PHP_EOL; } + static function foreach(){ echo __METHOD__, PHP_EOL; } + static function declare(){ echo __METHOD__, PHP_EOL; } + static function case(){ echo __METHOD__, PHP_EOL; } + static function do(){ echo __METHOD__, PHP_EOL; } + static function while(){ echo __METHOD__, PHP_EOL; } + static function as(){ echo __METHOD__, PHP_EOL; } + static function catch(){ echo __METHOD__, PHP_EOL; } + static function die(){ echo __METHOD__, PHP_EOL; } + static function self(){ echo __METHOD__, PHP_EOL; } + static function parent(){ echo __METHOD__, PHP_EOL; } +} + +Obj::empty(); +Obj::callable(); +Obj::trait(); +Obj::extends(); +Obj::implements(); +Obj::const(); +Obj::enddeclare(); +Obj::endfor(); +Obj::endforeach(); +Obj::endif(); +Obj::endwhile(); +Obj::and(); +Obj::global(); +Obj::goto(); +Obj::instanceof(); +Obj::insteadof(); +Obj::interface(); +Obj::new(); +Obj::or(); +Obj::xor(); +Obj::try(); +Obj::use(); +Obj::var(); +Obj::exit(); +Obj::list(); +Obj::clone(); +Obj::include(); +Obj::include_once(); +Obj::throw(); +Obj::array(); +Obj::print(); +Obj::echo(); +Obj::require(); +Obj::require_once(); +Obj::return(); +Obj::else(); +Obj::elseif(); +Obj::default(); +Obj::break(); +Obj::continue(); +Obj::switch(); +Obj::yield(); +Obj::function(); +Obj::if(); +Obj::endswitch(); +Obj::finally(); +Obj::for(); +Obj::foreach(); +Obj::declare(); +Obj::case(); +Obj::do(); +Obj::while(); +Obj::as(); +Obj::catch(); +Obj::die(); +Obj::self(); +Obj::parent(); + +echo "\nDone\n"; + +--EXPECTF-- +Obj::empty +Obj::callable +Obj::trait +Obj::extends +Obj::implements +Obj::const +Obj::enddeclare +Obj::endfor +Obj::endforeach +Obj::endif +Obj::endwhile +Obj::and +Obj::global +Obj::goto +Obj::instanceof +Obj::insteadof +Obj::interface +Obj::new +Obj::or +Obj::xor +Obj::try +Obj::use +Obj::var +Obj::exit +Obj::list +Obj::clone +Obj::include +Obj::include_once +Obj::throw +Obj::array +Obj::print +Obj::echo +Obj::require +Obj::require_once +Obj::return +Obj::else +Obj::elseif +Obj::default +Obj::break +Obj::continue +Obj::switch +Obj::yield +Obj::function +Obj::if +Obj::endswitch +Obj::finally +Obj::for +Obj::foreach +Obj::declare +Obj::case +Obj::do +Obj::while +Obj::as +Obj::catch +Obj::die +Obj::self +Obj::parent + +Done diff --git a/Zend/tests/grammar/semi_reserved_003.phpt b/Zend/tests/grammar/semi_reserved_003.phpt new file mode 100644 index 0000000000..fe2c44dc4c --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_003.phpt @@ -0,0 +1,210 @@ +--TEST-- +Test semi-reserved words as class properties +--FILE-- +<?php + +class Obj +{ + var $empty = 'empty'; + var $callable = 'callable'; + var $class = 'class'; + var $trait = 'trait'; + var $extends = 'extends'; + var $implements = 'implements'; + var $static = 'static'; + var $abstract = 'abstract'; + var $final = 'final'; + var $public = 'public'; + var $protected = 'protected'; + var $private = 'private'; + var $const = 'const'; + var $enddeclare = 'enddeclare'; + var $endfor = 'endfor'; + var $endforeach = 'endforeach'; + var $endif = 'endif'; + var $endwhile = 'endwhile'; + var $and = 'and'; + var $global = 'global'; + var $goto = 'goto'; + var $instanceof = 'instanceof'; + var $insteadof = 'insteadof'; + var $interface = 'interface'; + var $namespace = 'namespace'; + var $new = 'new'; + var $or = 'or'; + var $xor = 'xor'; + var $try = 'try'; + var $use = 'use'; + var $var = 'var'; + var $exit = 'exit'; + var $list = 'list'; + var $clone = 'clone'; + var $include = 'include'; + var $include_once = 'include_once'; + var $throw = 'throw'; + var $array = 'array'; + var $print = 'print'; + var $echo = 'echo'; + var $require = 'require'; + var $require_once = 'require_once'; + var $return = 'return'; + var $else = 'else'; + var $elseif = 'elseif'; + var $default = 'default'; + var $break = 'break'; + var $continue = 'continue'; + var $switch = 'switch'; + var $yield = 'yield'; + var $function = 'function'; + var $if = 'if'; + var $endswitch = 'endswitch'; + var $finally = 'finally'; + var $for = 'for'; + var $foreach = 'foreach'; + var $declare = 'declare'; + var $case = 'case'; + var $do = 'do'; + var $while = 'while'; + var $as = 'as'; + var $catch = 'catch'; + var $die = 'die'; + var $self = 'self'; +} + +$obj = new Obj; + +echo $obj->empty, PHP_EOL; +echo $obj->callable, PHP_EOL; +echo $obj->class, PHP_EOL; +echo $obj->trait, PHP_EOL; +echo $obj->extends, PHP_EOL; +echo $obj->implements, PHP_EOL; +echo $obj->static, PHP_EOL; +echo $obj->abstract, PHP_EOL; +echo $obj->final, PHP_EOL; +echo $obj->public, PHP_EOL; +echo $obj->protected, PHP_EOL; +echo $obj->private, PHP_EOL; +echo $obj->const, PHP_EOL; +echo $obj->enddeclare, PHP_EOL; +echo $obj->endfor, PHP_EOL; +echo $obj->endforeach, PHP_EOL; +echo $obj->endif, PHP_EOL; +echo $obj->endwhile, PHP_EOL; +echo $obj->and, PHP_EOL; +echo $obj->global, PHP_EOL; +echo $obj->goto, PHP_EOL; +echo $obj->instanceof, PHP_EOL; +echo $obj->insteadof, PHP_EOL; +echo $obj->interface, PHP_EOL; +echo $obj->namespace, PHP_EOL; +echo $obj->new, PHP_EOL; +echo $obj->or, PHP_EOL; +echo $obj->xor, PHP_EOL; +echo $obj->try, PHP_EOL; +echo $obj->use, PHP_EOL; +echo $obj->var, PHP_EOL; +echo $obj->exit, PHP_EOL; +echo $obj->list, PHP_EOL; +echo $obj->clone, PHP_EOL; +echo $obj->include, PHP_EOL; +echo $obj->include_once, PHP_EOL; +echo $obj->throw, PHP_EOL; +echo $obj->array, PHP_EOL; +echo $obj->print, PHP_EOL; +echo $obj->echo, PHP_EOL; +echo $obj->require, PHP_EOL; +echo $obj->require_once, PHP_EOL; +echo $obj->return, PHP_EOL; +echo $obj->else, PHP_EOL; +echo $obj->elseif, PHP_EOL; +echo $obj->default, PHP_EOL; +echo $obj->break, PHP_EOL; +echo $obj->continue, PHP_EOL; +echo $obj->switch, PHP_EOL; +echo $obj->yield, PHP_EOL; +echo $obj->function, PHP_EOL; +echo $obj->if, PHP_EOL; +echo $obj->endswitch, PHP_EOL; +echo $obj->finally, PHP_EOL; +echo $obj->for, PHP_EOL; +echo $obj->foreach, PHP_EOL; +echo $obj->declare, PHP_EOL; +echo $obj->case, PHP_EOL; +echo $obj->do, PHP_EOL; +echo $obj->while, PHP_EOL; +echo $obj->as, PHP_EOL; +echo $obj->catch, PHP_EOL; +echo $obj->die, PHP_EOL; +echo $obj->self, PHP_EOL; + +echo "\nDone\n"; + +?> +--EXPECTF-- +empty +callable +class +trait +extends +implements +static +abstract +final +public +protected +private +const +enddeclare +endfor +endforeach +endif +endwhile +and +global +goto +instanceof +insteadof +interface +namespace +new +or +xor +try +use +var +exit +list +clone +include +include_once +throw +array +print +echo +require +require_once +return +else +elseif +default +break +continue +switch +yield +function +if +endswitch +finally +for +foreach +declare +case +do +while +as +catch +die +self + +Done diff --git a/Zend/tests/grammar/semi_reserved_004.phpt b/Zend/tests/grammar/semi_reserved_004.phpt new file mode 100644 index 0000000000..40c5df14ef --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_004.phpt @@ -0,0 +1,210 @@ +--TEST-- +Test semi-reserved words as static class properties +--FILE-- +<?php + +class Obj +{ + static $empty = 'empty'; + static $callable = 'callable'; + static $class = 'class'; + static $trait = 'trait'; + static $extends = 'extends'; + static $implements = 'implements'; + static $static = 'static'; + static $abstract = 'abstract'; + static $final = 'final'; + static $public = 'public'; + static $protected = 'protected'; + static $private = 'private'; + static $const = 'const'; + static $enddeclare = 'enddeclare'; + static $endfor = 'endfor'; + static $endforeach = 'endforeach'; + static $endif = 'endif'; + static $endwhile = 'endwhile'; + static $and = 'and'; + static $global = 'global'; + static $goto = 'goto'; + static $instanceof = 'instanceof'; + static $insteadof = 'insteadof'; + static $interface = 'interface'; + static $namespace = 'namespace'; + static $new = 'new'; + static $or = 'or'; + static $xor = 'xor'; + static $try = 'try'; + static $use = 'use'; + static $var = 'var'; + static $exit = 'exit'; + static $list = 'list'; + static $clone = 'clone'; + static $include = 'include'; + static $include_once = 'include_once'; + static $throw = 'throw'; + static $array = 'array'; + static $print = 'print'; + static $echo = 'echo'; + static $require = 'require'; + static $require_once = 'require_once'; + static $return = 'return'; + static $else = 'else'; + static $elseif = 'elseif'; + static $default = 'default'; + static $break = 'break'; + static $continue = 'continue'; + static $switch = 'switch'; + static $yield = 'yield'; + static $function = 'function'; + static $if = 'if'; + static $endswitch = 'endswitch'; + static $finally = 'finally'; + static $for = 'for'; + static $foreach = 'foreach'; + static $declare = 'declare'; + static $case = 'case'; + static $do = 'do'; + static $while = 'while'; + static $as = 'as'; + static $catch = 'catch'; + static $die = 'die'; + static $self = 'self'; + static $parent = 'parent'; +} + +echo Obj::$empty, PHP_EOL; +echo Obj::$callable, PHP_EOL; +echo Obj::$class, PHP_EOL; +echo Obj::$trait, PHP_EOL; +echo Obj::$extends, PHP_EOL; +echo Obj::$implements, PHP_EOL; +echo Obj::$static, PHP_EOL; +echo Obj::$abstract, PHP_EOL; +echo Obj::$final, PHP_EOL; +echo Obj::$public, PHP_EOL; +echo Obj::$protected, PHP_EOL; +echo Obj::$private, PHP_EOL; +echo Obj::$const, PHP_EOL; +echo Obj::$enddeclare, PHP_EOL; +echo Obj::$endfor, PHP_EOL; +echo Obj::$endforeach, PHP_EOL; +echo Obj::$endif, PHP_EOL; +echo Obj::$endwhile, PHP_EOL; +echo Obj::$and, PHP_EOL; +echo Obj::$global, PHP_EOL; +echo Obj::$goto, PHP_EOL; +echo Obj::$instanceof, PHP_EOL; +echo Obj::$insteadof, PHP_EOL; +echo Obj::$interface, PHP_EOL; +echo Obj::$namespace, PHP_EOL; +echo Obj::$new, PHP_EOL; +echo Obj::$or, PHP_EOL; +echo Obj::$xor, PHP_EOL; +echo Obj::$try, PHP_EOL; +echo Obj::$use, PHP_EOL; +echo Obj::$var, PHP_EOL; +echo Obj::$exit, PHP_EOL; +echo Obj::$list, PHP_EOL; +echo Obj::$clone, PHP_EOL; +echo Obj::$include, PHP_EOL; +echo Obj::$include_once, PHP_EOL; +echo Obj::$throw, PHP_EOL; +echo Obj::$array, PHP_EOL; +echo Obj::$print, PHP_EOL; +echo Obj::$echo, PHP_EOL; +echo Obj::$require, PHP_EOL; +echo Obj::$require_once, PHP_EOL; +echo Obj::$return, PHP_EOL; +echo Obj::$else, PHP_EOL; +echo Obj::$elseif, PHP_EOL; +echo Obj::$default, PHP_EOL; +echo Obj::$break, PHP_EOL; +echo Obj::$continue, PHP_EOL; +echo Obj::$switch, PHP_EOL; +echo Obj::$yield, PHP_EOL; +echo Obj::$function, PHP_EOL; +echo Obj::$if, PHP_EOL; +echo Obj::$endswitch, PHP_EOL; +echo Obj::$finally, PHP_EOL; +echo Obj::$for, PHP_EOL; +echo Obj::$foreach, PHP_EOL; +echo Obj::$declare, PHP_EOL; +echo Obj::$case, PHP_EOL; +echo Obj::$do, PHP_EOL; +echo Obj::$while, PHP_EOL; +echo Obj::$as, PHP_EOL; +echo Obj::$catch, PHP_EOL; +echo Obj::$die, PHP_EOL; +echo Obj::$self, PHP_EOL; +echo Obj::$parent, PHP_EOL; + +echo "\nDone\n"; + +--EXPECTF-- +empty +callable +class +trait +extends +implements +static +abstract +final +public +protected +private +const +enddeclare +endfor +endforeach +endif +endwhile +and +global +goto +instanceof +insteadof +interface +namespace +new +or +xor +try +use +var +exit +list +clone +include +include_once +throw +array +print +echo +require +require_once +return +else +elseif +default +break +continue +switch +yield +function +if +endswitch +finally +for +foreach +declare +case +do +while +as +catch +die +self +parent + +Done diff --git a/Zend/tests/grammar/semi_reserved_005.phpt b/Zend/tests/grammar/semi_reserved_005.phpt new file mode 100644 index 0000000000..dcd4018d71 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_005.phpt @@ -0,0 +1,189 @@ +--TEST-- +Test semi-reserved words as class constants +--FILE-- +<?php + +class Obj +{ + const EMPTY = 'empty'; + const CALLABLE = 'callable'; + const TRAIT = 'trait'; + const EXTENDS = 'extends'; + const IMPLEMENTS = 'implements'; + const CONST = 'const'; + const ENDDECLARE = 'enddeclare'; + const ENDFOR = 'endfor'; + const ENDFOREACH = 'endforeach'; + const ENDIF = 'endif'; + const ENDWHILE = 'endwhile'; + const AND = 'and'; + const GLOBAL = 'global'; + const GOTO = 'goto'; + const INSTANCEOF = 'instanceof'; + const INSTEADOF = 'insteadof'; + const INTERFACE = 'interface'; + const NAMESPACE = 'namespace'; + const NEW = 'new'; + const OR = 'or'; + const XOR = 'xor'; + const TRY = 'try'; + const USE = 'use'; + const VAR = 'var'; + const EXIT = 'exit'; + const LIST = 'list'; + const CLONE = 'clone'; + const INCLUDE = 'include'; + const INCLUDE_ONCE = 'include_once'; + const THROW = 'throw'; + const ARRAY = 'array'; + const PRINT = 'print'; + const ECHO = 'echo'; + const REQUIRE = 'require'; + const REQUIRE_ONCE = 'require_once'; + const RETURN = 'return'; + const ELSE = 'else'; + const ELSEIF = 'elseif'; + const DEFAULT = 'default'; + const BREAK = 'break'; + const CONTINUE = 'continue'; + const SWITCH = 'switch'; + const YIELD = 'yield'; + const FUNCTION = 'function'; + const IF = 'if'; + const ENDSWITCH = 'endswitch'; + const FINALLY = 'finally'; + const FOR = 'for'; + const FOREACH = 'foreach'; + const DECLARE = 'declare'; + const CASE = 'case'; + const DO = 'do'; + const WHILE = 'while'; + const AS = 'as'; + const CATCH = 'catch'; + const DIE = 'die'; + const SELF = 'self'; + const PARENT = 'parent'; +} + +echo Obj::EMPTY, PHP_EOL; +echo Obj::CALLABLE, PHP_EOL; +echo Obj::TRAIT, PHP_EOL; +echo Obj::EXTENDS, PHP_EOL; +echo Obj::IMPLEMENTS, PHP_EOL; +echo Obj::CONST, PHP_EOL; +echo Obj::ENDDECLARE, PHP_EOL; +echo Obj::ENDFOR, PHP_EOL; +echo Obj::ENDFOREACH, PHP_EOL; +echo Obj::ENDIF, PHP_EOL; +echo Obj::ENDWHILE, PHP_EOL; +echo Obj::AND, PHP_EOL; +echo Obj::GLOBAL, PHP_EOL; +echo Obj::GOTO, PHP_EOL; +echo Obj::INSTANCEOF, PHP_EOL; +echo Obj::INSTEADOF, PHP_EOL; +echo Obj::INTERFACE, PHP_EOL; +echo Obj::NAMESPACE, PHP_EOL; +echo Obj::NEW, PHP_EOL; +echo Obj::OR, PHP_EOL; +echo Obj::XOR, PHP_EOL; +echo Obj::TRY, PHP_EOL; +echo Obj::USE, PHP_EOL; +echo Obj::VAR, PHP_EOL; +echo Obj::EXIT, PHP_EOL; +echo Obj::LIST, PHP_EOL; +echo Obj::CLONE, PHP_EOL; +echo Obj::INCLUDE, PHP_EOL; +echo Obj::INCLUDE_ONCE, PHP_EOL; +echo Obj::THROW, PHP_EOL; +echo Obj::ARRAY, PHP_EOL; +echo Obj::PRINT, PHP_EOL; +echo Obj::ECHO, PHP_EOL; +echo Obj::REQUIRE, PHP_EOL; +echo Obj::REQUIRE_ONCE, PHP_EOL; +echo Obj::RETURN, PHP_EOL; +echo Obj::ELSE, PHP_EOL; +echo Obj::ELSEIF, PHP_EOL; +echo Obj::DEFAULT, PHP_EOL; +echo Obj::BREAK, PHP_EOL; +echo Obj::CONTINUE, PHP_EOL; +echo Obj::SWITCH, PHP_EOL; +echo Obj::YIELD, PHP_EOL; +echo Obj::FUNCTION, PHP_EOL; +echo Obj::IF, PHP_EOL; +echo Obj::ENDSWITCH, PHP_EOL; +echo Obj::FINALLY, PHP_EOL; +echo Obj::FOR, PHP_EOL; +echo Obj::FOREACH, PHP_EOL; +echo Obj::DECLARE, PHP_EOL; +echo Obj::CASE, PHP_EOL; +echo Obj::DO, PHP_EOL; +echo Obj::WHILE, PHP_EOL; +echo Obj::AS, PHP_EOL; +echo Obj::CATCH, PHP_EOL; +echo Obj::DIE, PHP_EOL; +echo Obj::SELF, PHP_EOL; +echo Obj::PARENT, PHP_EOL; + +echo "\nDone\n"; + +--EXPECTF-- +empty +callable +trait +extends +implements +const +enddeclare +endfor +endforeach +endif +endwhile +and +global +goto +instanceof +insteadof +interface +namespace +new +or +xor +try +use +var +exit +list +clone +include +include_once +throw +array +print +echo +require +require_once +return +else +elseif +default +break +continue +switch +yield +function +if +endswitch +finally +for +foreach +declare +case +do +while +as +catch +die +self +parent + +Done diff --git a/Zend/tests/grammar/semi_reserved_006.phpt b/Zend/tests/grammar/semi_reserved_006.phpt new file mode 100644 index 0000000000..334d09ac36 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_006.phpt @@ -0,0 +1,80 @@ +--TEST-- +Test semi-reserved method and constant names and trait conflict resolution +--FILE-- +<?php + +trait TraitA +{ + public function catch(){ echo __METHOD__, PHP_EOL; } + private function list(){ echo __METHOD__, PHP_EOL; } +} + +trait TraitB +{ + static $list = ['a' => ['b' => ['c']]]; + + public static function catch(){ echo __METHOD__, PHP_EOL; } + private static function throw(){ echo __METHOD__, PHP_EOL; } + private static function self(){ echo __METHOD__, PHP_EOL; } +} + +trait TraitC +{ + public static function exit(){ echo __METHOD__, PHP_EOL; } + protected static function try(){ echo __METHOD__, PHP_EOL; } +} + +class Foo +{ + use TraitA, TraitB { + TraitA + :: + catch insteadof namespace\TraitB; + TraitA::list as public foreach; + TraitB::throw as public; + TraitB::self as public; + } + + use TraitC { + try as public attempt; + exit as die; + \TraitC::exit as bye; + namespace\TraitC::exit as byebye; + TraitC + :: + exit as farewell; + } +} + +(new Foo)->catch(); +(new Foo)->foreach(); +Foo::throw(); +Foo::self(); +var_dump(Foo::$list['a']); +Foo::attempt(); +Foo::die(); +Foo::bye(); +Foo::byebye(); +Foo::farewell(); + +echo "\nDone\n"; + +--EXPECTF-- +TraitA::catch +TraitA::list +TraitB::throw +TraitB::self +array(1) { + ["b"]=> + array(1) { + [0]=> + string(1) "c" + } +} +TraitC::try +TraitC::exit +TraitC::exit +TraitC::exit +TraitC::exit + +Done diff --git a/Zend/tests/grammar/semi_reserved_007.phpt b/Zend/tests/grammar/semi_reserved_007.phpt new file mode 100644 index 0000000000..5105629cbe --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_007.phpt @@ -0,0 +1,37 @@ +--TEST-- +Edge case: self::self, self::parent, parent::self semi reserved constants access +--FILE-- +<?php + +class Foo { + const self = "self"; + const parent = "parent"; + public function __construct() { + echo "From ", __METHOD__, ":", PHP_EOL; + echo self::self, PHP_EOL; + echo self::parent, PHP_EOL; + } +} + +class Bar extends Foo { + public function __construct() { + parent::__construct(); + echo "From ", __METHOD__, ":", PHP_EOL; + echo parent::self, PHP_EOL; + echo parent::parent, PHP_EOL; + } +} + +new Bar; + +echo "\nDone\n"; + +--EXPECTF-- +From Foo::__construct: +self +parent +From Bar::__construct: +self +parent + +Done
\ No newline at end of file diff --git a/Zend/tests/grammar/semi_reserved_008.phpt b/Zend/tests/grammar/semi_reserved_008.phpt new file mode 100644 index 0000000000..43218b1b05 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_008.phpt @@ -0,0 +1,68 @@ +--TEST-- +Testing with comments around semi-reserved names (not intended to be legible) +--FILE-- +<?php + +trait TraitA +{ + public static function list(){ echo __METHOD__, PHP_EOL; } + public static function /* comment */ catch(){ echo __METHOD__, PHP_EOL; } + private static function // comment + throw(){ echo __METHOD__, PHP_EOL; } + private static function + # comment + self(){ echo __METHOD__, PHP_EOL; } +} + +trait TraitB +{ + public static function exit(){ echo __METHOD__, PHP_EOL; } + protected static function try(){ echo __METHOD__, PHP_EOL; } +} + +class Foo +{ + use TraitA { + TraitA:: + // + /** doc comment */ + # + catch /* comment */ + // comment + # comment + insteadof TraitB; + + TraitA::list as public /**/ foreach; + } + + use TraitB { + try /*comment*/ as public attempt; + exit // comment + as/*comment*/die; // non qualified + \TraitB::exit as bye; // full qualified + namespace\TraitB::exit # + as byebye; // even more full qualified + TraitB + :: + /** */ + exit as farewell; // full qualified with weird spacing + } +} + +Foo /**/ +# +// +/** */ +:: +/**/ +# +// +/** */ +attempt(); + +echo PHP_EOL, "Done", PHP_EOL; + +--EXPECTF-- +TraitB::try + +Done diff --git a/Zend/tests/grammar/semi_reserved_009.phpt b/Zend/tests/grammar/semi_reserved_009.phpt new file mode 100644 index 0000000000..1a7b0fc371 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_009.phpt @@ -0,0 +1,25 @@ +--TEST-- +Edge case: T_STRING<as> as T_STRING<?> +--FILE-- +<?php + +trait TraitA +{ + public static function as(){ echo __METHOD__, PHP_EOL; } +} + +class Foo +{ + use TraitA { + as as try; + } +} + +Foo::try(); + +echo PHP_EOL, "Done", PHP_EOL; + +--EXPECTF-- +TraitA::as + +Done diff --git a/Zend/tests/grammar/semi_reserved_010.phpt b/Zend/tests/grammar/semi_reserved_010.phpt new file mode 100644 index 0000000000..508a7867a4 --- /dev/null +++ b/Zend/tests/grammar/semi_reserved_010.phpt @@ -0,0 +1,31 @@ +--TEST-- +Edge case: T_STRING<insteadof> insteadof T_STRING<?> +--FILE-- +<?php + +trait TraitA +{ + public static function insteadof(){ echo __METHOD__, PHP_EOL; } +} + +trait TraitB +{ + public static function insteadof(){ echo __METHOD__, PHP_EOL; } +} + +class Foo +{ + use TraitA , TraitB { + TraitB::insteadof + insteadof TraitA; + } +} + +Foo::insteadof(); + +echo PHP_EOL, "Done", PHP_EOL; + +--EXPECTF-- +TraitB::insteadof + +Done diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 4721ecd939..a6dc298cb0 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -30,7 +30,6 @@ #include "zend_interfaces.h" #include "zend_virtual_cwd.h" #include "zend_multibyte.h" -#include "zend_language_scanner.h" #include "zend_inheritance.h" #define SET_NODE(target, src) do { \ @@ -557,7 +556,10 @@ static int zend_add_const_name_literal(zend_op_array *op_array, zend_string *nam op.constant = zend_add_literal(CG(active_op_array), &_c); \ } while (0) -void zend_stop_lexing(void) { +void zend_stop_lexing(void) +{ + if(LANG_SCNG(on_event)) LANG_SCNG(on_event)(ON_STOP, END, 0); + LANG_SCNG(yy_cursor) = LANG_SCNG(yy_limit); } diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 326955a103..28487a2a4a 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -249,6 +249,12 @@ struct _zend_ini_scanner_globals { int scanner_mode; }; +typedef enum { + ON_TOKEN, + ON_FEEDBACK, + ON_STOP +} zend_php_scanner_event; + struct _zend_php_scanner_globals { zend_file_handle *yy_in; zend_file_handle *yy_out; @@ -278,6 +284,9 @@ struct _zend_php_scanner_globals { /* initial string length after scanning to first variable */ int scanned_string_len; + + /* hooks */ + void (* on_event)(zend_php_scanner_event event, int token, int line); }; #endif /* ZEND_GLOBALS_H */ diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y index 2541c9f571..f6318ec0c0 100644 --- a/Zend/zend_language_parser.y +++ b/Zend/zend_language_parser.y @@ -35,6 +35,7 @@ #include "zend_globals.h" #include "zend_API.h" #include "zend_constants.h" +#include "zend_language_scanner.h" #define YYSIZE_T size_t #define yytnamerr zend_yytnamerr @@ -243,7 +244,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*); %type <ast> absolute_trait_method_reference trait_method_reference property echo_expr %type <ast> new_expr anonymous_class class_name class_name_reference simple_variable %type <ast> internal_functions_in_yacc -%type <ast> exit_expr scalar backticks_expr lexical_var function_call member_name +%type <ast> exit_expr scalar backticks_expr lexical_var function_call member_name property_name %type <ast> variable_class_name dereferencable_scalar class_name_scalar constant dereferencable %type <ast> callable_expr callable_variable static_member new_variable %type <ast> assignment_list_element array_pair encaps_var encaps_var_offset isset_variables @@ -252,10 +253,11 @@ static YYSIZE_T zend_yytnamerr(char*, const char*); %type <ast> echo_expr_list unset_variables catch_list parameter_list class_statement_list %type <ast> implements_list case_list if_stmt_without_else %type <ast> non_empty_parameter_list argument_list non_empty_argument_list property_list -%type <ast> class_const_list name_list trait_adaptations method_body non_empty_for_exprs +%type <ast> class_const_list class_const_decl name_list trait_adaptations method_body non_empty_for_exprs %type <ast> ctor_arguments alt_if_stmt_without_else trait_adaptation_list lexical_vars %type <ast> lexical_var_list encaps_list array_pair_list non_empty_array_pair_list %type <ast> assignment_list isset_variable type return_type +%type <ast> identifier %type <num> returns_ref function is_reference is_variadic variable_modifiers %type <num> method_modifiers trait_modifiers non_empty_member_modifiers member_modifier @@ -269,6 +271,26 @@ start: top_statement_list { CG(ast) = $1; } ; +semi_reserved: + T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND + | T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE | T_ENDWHILE + | T_FOR | T_ENDFOR | T_FOREACH | T_ENDFOREACH | T_DECLARE | T_ENDDECLARE | T_AS | T_TRY | T_CATCH | T_FINALLY + | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO + | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT | T_BREAK + | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE + // | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC + // | T_CLASS +; + +identifier: + T_STRING { $$ = $1; } + | semi_reserved { + zval zv; + zend_lex_tstring(&zv); + $$ = zend_ast_create_zval(&zv); + } +; + top_statement_list: top_statement_list top_statement { $$ = zend_ast_list_add($1, $2); } | /* empty */ { $$ = zend_ast_create_list(0, ZEND_AST_STMT_LIST); } @@ -673,7 +695,7 @@ class_statement: { $$ = $2; RESET_DOC_COMMENT(); } | T_USE name_list trait_adaptations { $$ = zend_ast_create(ZEND_AST_USE_TRAIT, $2, $3); } - | method_modifiers function returns_ref T_STRING '(' parameter_list ')' + | method_modifiers function returns_ref identifier '(' parameter_list ')' return_type backup_doc_comment method_body { $$ = zend_ast_create_decl(ZEND_AST_METHOD, $3 | $1, $2, $9, zend_ast_get_str($4), $6, NULL, $10, $8); } @@ -708,20 +730,20 @@ trait_precedence: ; trait_alias: - trait_method_reference T_AS trait_modifiers T_STRING + trait_method_reference T_AS trait_modifiers identifier { $$ = zend_ast_create_ex(ZEND_AST_TRAIT_ALIAS, $3, $1, $4); } | trait_method_reference T_AS member_modifier { $$ = zend_ast_create_ex(ZEND_AST_TRAIT_ALIAS, $3, $1, NULL); } ; trait_method_reference: - T_STRING + identifier { $$ = zend_ast_create(ZEND_AST_METHOD_REFERENCE, NULL, $1); } | absolute_trait_method_reference { $$ = $1; } ; absolute_trait_method_reference: - name T_PAAMAYIM_NEKUDOTAYIM T_STRING + name T_PAAMAYIM_NEKUDOTAYIM identifier { $$ = zend_ast_create(ZEND_AST_METHOD_REFERENCE, $1, $3); } ; @@ -773,8 +795,12 @@ property: ; class_const_list: - class_const_list ',' const_decl { $$ = zend_ast_list_add($1, $3); } - | const_decl { $$ = zend_ast_create_list(1, ZEND_AST_CLASS_CONST_DECL, $1); } + class_const_list ',' class_const_decl { $$ = zend_ast_list_add($1, $3); } + | class_const_decl { $$ = zend_ast_create_list(1, ZEND_AST_CLASS_CONST_DECL, $1); } +; + +class_const_decl: + identifier '=' expr { $$ = zend_ast_create(ZEND_AST_CONST_ELEM, $1, $3); } ; const_decl: @@ -1034,9 +1060,9 @@ scalar: constant: name { $$ = zend_ast_create(ZEND_AST_CONST, $1); } - | class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING + | class_name T_PAAMAYIM_NEKUDOTAYIM identifier { $$ = zend_ast_create(ZEND_AST_CLASS_CONST, $1, $3); } - | variable_class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING + | variable_class_name T_PAAMAYIM_NEKUDOTAYIM identifier { $$ = zend_ast_create(ZEND_AST_CLASS_CONST, $1, $3); } ; @@ -1080,7 +1106,7 @@ callable_variable: { $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); } | dereferencable '{' expr '}' { $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); } - | dereferencable T_OBJECT_OPERATOR member_name argument_list + | dereferencable T_OBJECT_OPERATOR property_name argument_list { $$ = zend_ast_create(ZEND_AST_METHOD_CALL, $1, $3, $4); } | function_call { $$ = $1; } ; @@ -1090,7 +1116,7 @@ variable: { $$ = $1; } | static_member { $$ = $1; } - | dereferencable T_OBJECT_OPERATOR member_name + | dereferencable T_OBJECT_OPERATOR property_name { $$ = zend_ast_create(ZEND_AST_PROP, $1, $3); } ; @@ -1114,7 +1140,7 @@ new_variable: { $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); } | new_variable '{' expr '}' { $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); } - | new_variable T_OBJECT_OPERATOR member_name + | new_variable T_OBJECT_OPERATOR property_name { $$ = zend_ast_create(ZEND_AST_PROP, $1, $3); } | class_name T_PAAMAYIM_NEKUDOTAYIM simple_variable { $$ = zend_ast_create(ZEND_AST_STATIC_PROP, $1, $3); } @@ -1123,7 +1149,13 @@ new_variable: ; member_name: - T_STRING { $$ = $1; } + identifier { $$ = $1; } + | '{' expr '}' { $$ = $2; } + | simple_variable { $$ = zend_ast_create(ZEND_AST_VAR, $1); } +; + +property_name: + T_STRING { $$ = $1; } | '{' expr '}' { $$ = $2; } | simple_variable { $$ = zend_ast_create(ZEND_AST_VAR, $1); } ; diff --git a/Zend/zend_language_scanner.h b/Zend/zend_language_scanner.h index c82b3069c5..3b75ff8cc4 100644 --- a/Zend/zend_language_scanner.h +++ b/Zend/zend_language_scanner.h @@ -50,6 +50,9 @@ typedef struct _zend_lex_state { zend_encoding_filter output_filter; const zend_encoding *script_encoding; + /* hooks */ + void (* on_event)(zend_php_scanner_event event, int token, int line); + zend_ast *ast; zend_arena *ast_arena; } zend_lex_state; @@ -66,6 +69,7 @@ ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state); ZEND_API int zend_prepare_string_for_scanning(zval *str, char *filename); ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter, const zend_encoding *old_encoding); ZEND_API int zend_multibyte_set_filter(const zend_encoding *onetime_encoding); +ZEND_API void zend_lex_tstring(zval *zv); END_EXTERN_C() diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l index fdba4b9f07..cde0621df0 100644 --- a/Zend/zend_language_scanner.l +++ b/Zend/zend_language_scanner.l @@ -193,6 +193,7 @@ void shutdown_scanner(void) zend_stack_destroy(&SCNG(state_stack)); zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1); zend_ptr_stack_destroy(&SCNG(heredoc_label_stack)); + SCNG(on_event) = NULL; } ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state) @@ -223,6 +224,8 @@ ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state) lex_state->output_filter = SCNG(output_filter); lex_state->script_encoding = SCNG(script_encoding); + lex_state->on_event = SCNG(on_event); + lex_state->ast = CG(ast); lex_state->ast_arena = CG(ast_arena); } @@ -260,6 +263,8 @@ ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state) SCNG(output_filter) = lex_state->output_filter; SCNG(script_encoding) = lex_state->script_encoding; + SCNG(on_event) = lex_state->on_event; + CG(ast) = lex_state->ast; CG(ast_arena) = lex_state->ast_arena; @@ -276,6 +281,13 @@ ZEND_API void zend_destroy_file_handle(zend_file_handle *file_handle) } } +ZEND_API void zend_lex_tstring(zval *zv) +{ + if (SCNG(on_event)) SCNG(on_event)(ON_FEEDBACK, T_STRING, 0); + + ZVAL_STRINGL(zv, (char*)SCNG(yy_text), SCNG(yy_leng)); +} + #define BOM_UTF32_BE "\x00\x00\xfe\xff" #define BOM_UTF32_LE "\xff\xfe\x00\x00" #define BOM_UTF16_BE "\xfe\xff" @@ -1083,9 +1095,20 @@ static int zend_scan_escape_string(zval *zendlval, char *str, int len, char quot return SUCCESS; } +static zend_always_inline int emit_token(int token, int token_line) +{ + if(SCNG(on_event)) SCNG(on_event)(ON_TOKEN, token, token_line); + + return token; +} + +#define RETURN_TOKEN(token) return emit_token(token, start_line); int lex_scan(zval *zendlval) { + +int start_line = CG(zend_lineno); + restart: SCNG(yy_text) = YYCURSOR; @@ -1107,183 +1130,183 @@ NEWLINE ("\r"|"\n"|"\r\n") <!*> := yyleng = YYCURSOR - SCNG(yy_text); <ST_IN_SCRIPTING>"exit" { - return T_EXIT; + RETURN_TOKEN(T_EXIT); } <ST_IN_SCRIPTING>"die" { - return T_EXIT; + RETURN_TOKEN(T_EXIT); } <ST_IN_SCRIPTING>"function" { - return T_FUNCTION; + RETURN_TOKEN(T_FUNCTION); } <ST_IN_SCRIPTING>"const" { - return T_CONST; + RETURN_TOKEN(T_CONST); } <ST_IN_SCRIPTING>"return" { - return T_RETURN; + RETURN_TOKEN(T_RETURN); } <ST_IN_SCRIPTING>"yield"{WHITESPACE}"from" { - return T_YIELD_FROM; + RETURN_TOKEN(T_YIELD_FROM); } <ST_IN_SCRIPTING>"yield" { - return T_YIELD; + RETURN_TOKEN(T_YIELD); } <ST_IN_SCRIPTING>"try" { - return T_TRY; + RETURN_TOKEN(T_TRY); } <ST_IN_SCRIPTING>"catch" { - return T_CATCH; + RETURN_TOKEN(T_CATCH); } <ST_IN_SCRIPTING>"finally" { - return T_FINALLY; + RETURN_TOKEN(T_FINALLY); } <ST_IN_SCRIPTING>"throw" { - return T_THROW; + RETURN_TOKEN(T_THROW); } <ST_IN_SCRIPTING>"if" { - return T_IF; + RETURN_TOKEN(T_IF); } <ST_IN_SCRIPTING>"elseif" { - return T_ELSEIF; + RETURN_TOKEN(T_ELSEIF); } <ST_IN_SCRIPTING>"endif" { - return T_ENDIF; + RETURN_TOKEN(T_ENDIF); } <ST_IN_SCRIPTING>"else" { - return T_ELSE; + RETURN_TOKEN(T_ELSE); } <ST_IN_SCRIPTING>"while" { - return T_WHILE; + RETURN_TOKEN(T_WHILE); } <ST_IN_SCRIPTING>"endwhile" { - return T_ENDWHILE; + RETURN_TOKEN(T_ENDWHILE); } <ST_IN_SCRIPTING>"do" { - return T_DO; + RETURN_TOKEN(T_DO); } <ST_IN_SCRIPTING>"for" { - return T_FOR; + RETURN_TOKEN(T_FOR); } <ST_IN_SCRIPTING>"endfor" { - return T_ENDFOR; + RETURN_TOKEN(T_ENDFOR); } <ST_IN_SCRIPTING>"foreach" { - return T_FOREACH; + RETURN_TOKEN(T_FOREACH); } <ST_IN_SCRIPTING>"endforeach" { - return T_ENDFOREACH; + RETURN_TOKEN(T_ENDFOREACH); } <ST_IN_SCRIPTING>"declare" { - return T_DECLARE; + RETURN_TOKEN(T_DECLARE); } <ST_IN_SCRIPTING>"enddeclare" { - return T_ENDDECLARE; + RETURN_TOKEN(T_ENDDECLARE); } <ST_IN_SCRIPTING>"instanceof" { - return T_INSTANCEOF; + RETURN_TOKEN(T_INSTANCEOF); } <ST_IN_SCRIPTING>"as" { - return T_AS; + RETURN_TOKEN(T_AS); } <ST_IN_SCRIPTING>"switch" { - return T_SWITCH; + RETURN_TOKEN(T_SWITCH); } <ST_IN_SCRIPTING>"endswitch" { - return T_ENDSWITCH; + RETURN_TOKEN(T_ENDSWITCH); } <ST_IN_SCRIPTING>"case" { - return T_CASE; + RETURN_TOKEN(T_CASE); } <ST_IN_SCRIPTING>"default" { - return T_DEFAULT; + RETURN_TOKEN(T_DEFAULT); } <ST_IN_SCRIPTING>"break" { - return T_BREAK; + RETURN_TOKEN(T_BREAK); } <ST_IN_SCRIPTING>"continue" { - return T_CONTINUE; + RETURN_TOKEN(T_CONTINUE); } <ST_IN_SCRIPTING>"goto" { - return T_GOTO; + RETURN_TOKEN(T_GOTO); } <ST_IN_SCRIPTING>"echo" { - return T_ECHO; + RETURN_TOKEN(T_ECHO); } <ST_IN_SCRIPTING>"print" { - return T_PRINT; + RETURN_TOKEN(T_PRINT); } <ST_IN_SCRIPTING>"class" { - return T_CLASS; + RETURN_TOKEN(T_CLASS); } <ST_IN_SCRIPTING>"interface" { - return T_INTERFACE; + RETURN_TOKEN(T_INTERFACE); } <ST_IN_SCRIPTING>"trait" { - return T_TRAIT; + RETURN_TOKEN(T_TRAIT); } <ST_IN_SCRIPTING>"extends" { - return T_EXTENDS; + RETURN_TOKEN(T_EXTENDS); } <ST_IN_SCRIPTING>"implements" { - return T_IMPLEMENTS; + RETURN_TOKEN(T_IMPLEMENTS); } <ST_IN_SCRIPTING>"->" { yy_push_state(ST_LOOKING_FOR_PROPERTY); - return T_OBJECT_OPERATOR; + RETURN_TOKEN(T_OBJECT_OPERATOR); } <ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>{WHITESPACE}+ { HANDLE_NEWLINES(yytext, yyleng); - return T_WHITESPACE; + RETURN_TOKEN(T_WHITESPACE); } <ST_LOOKING_FOR_PROPERTY>"->" { - return T_OBJECT_OPERATOR; + RETURN_TOKEN(T_OBJECT_OPERATOR); } <ST_LOOKING_FOR_PROPERTY>{LABEL} { yy_pop_state(); zend_copy_value(zendlval, yytext, yyleng); - return T_STRING; + RETURN_TOKEN(T_STRING); } <ST_LOOKING_FOR_PROPERTY>{ANY_CHAR} { @@ -1293,283 +1316,283 @@ NEWLINE ("\r"|"\n"|"\r\n") } <ST_IN_SCRIPTING>"::" { - return T_PAAMAYIM_NEKUDOTAYIM; + RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM); } <ST_IN_SCRIPTING>"\\" { - return T_NS_SEPARATOR; + RETURN_TOKEN(T_NS_SEPARATOR); } <ST_IN_SCRIPTING>"..." { - return T_ELLIPSIS; + RETURN_TOKEN(T_ELLIPSIS); } <ST_IN_SCRIPTING>"??" { - return T_COALESCE; + RETURN_TOKEN(T_COALESCE); } <ST_IN_SCRIPTING>"new" { - return T_NEW; + RETURN_TOKEN(T_NEW); } <ST_IN_SCRIPTING>"clone" { - return T_CLONE; + RETURN_TOKEN(T_CLONE); } <ST_IN_SCRIPTING>"var" { - return T_VAR; + RETURN_TOKEN(T_VAR); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("int"|"integer"){TABS_AND_SPACES}")" { - return T_INT_CAST; + RETURN_TOKEN(T_INT_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("real"|"double"|"float"){TABS_AND_SPACES}")" { - return T_DOUBLE_CAST; + RETURN_TOKEN(T_DOUBLE_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("string"|"binary"){TABS_AND_SPACES}")" { - return T_STRING_CAST; + RETURN_TOKEN(T_STRING_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"array"{TABS_AND_SPACES}")" { - return T_ARRAY_CAST; + RETURN_TOKEN(T_ARRAY_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"object"{TABS_AND_SPACES}")" { - return T_OBJECT_CAST; + RETURN_TOKEN(T_OBJECT_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("bool"|"boolean"){TABS_AND_SPACES}")" { - return T_BOOL_CAST; + RETURN_TOKEN(T_BOOL_CAST); } <ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("unset"){TABS_AND_SPACES}")" { - return T_UNSET_CAST; + RETURN_TOKEN(T_UNSET_CAST); } <ST_IN_SCRIPTING>"eval" { - return T_EVAL; + RETURN_TOKEN(T_EVAL); } <ST_IN_SCRIPTING>"include" { - return T_INCLUDE; + RETURN_TOKEN(T_INCLUDE); } <ST_IN_SCRIPTING>"include_once" { - return T_INCLUDE_ONCE; + RETURN_TOKEN(T_INCLUDE_ONCE); } <ST_IN_SCRIPTING>"require" { - return T_REQUIRE; + RETURN_TOKEN(T_REQUIRE); } <ST_IN_SCRIPTING>"require_once" { - return T_REQUIRE_ONCE; + RETURN_TOKEN(T_REQUIRE_ONCE); } <ST_IN_SCRIPTING>"namespace" { - return T_NAMESPACE; + RETURN_TOKEN(T_NAMESPACE); } <ST_IN_SCRIPTING>"use" { - return T_USE; + RETURN_TOKEN(T_USE); } <ST_IN_SCRIPTING>"insteadof" { - return T_INSTEADOF; + RETURN_TOKEN(T_INSTEADOF); } <ST_IN_SCRIPTING>"global" { - return T_GLOBAL; + RETURN_TOKEN(T_GLOBAL); } <ST_IN_SCRIPTING>"isset" { - return T_ISSET; + RETURN_TOKEN(T_ISSET); } <ST_IN_SCRIPTING>"empty" { - return T_EMPTY; + RETURN_TOKEN(T_EMPTY); } <ST_IN_SCRIPTING>"__halt_compiler" { - return T_HALT_COMPILER; + RETURN_TOKEN(T_HALT_COMPILER); } <ST_IN_SCRIPTING>"static" { - return T_STATIC; + RETURN_TOKEN(T_STATIC); } <ST_IN_SCRIPTING>"abstract" { - return T_ABSTRACT; + RETURN_TOKEN(T_ABSTRACT); } <ST_IN_SCRIPTING>"final" { - return T_FINAL; + RETURN_TOKEN(T_FINAL); } <ST_IN_SCRIPTING>"private" { - return T_PRIVATE; + RETURN_TOKEN(T_PRIVATE); } <ST_IN_SCRIPTING>"protected" { - return T_PROTECTED; + RETURN_TOKEN(T_PROTECTED); } <ST_IN_SCRIPTING>"public" { - return T_PUBLIC; + RETURN_TOKEN(T_PUBLIC); } <ST_IN_SCRIPTING>"unset" { - return T_UNSET; + RETURN_TOKEN(T_UNSET); } <ST_IN_SCRIPTING>"=>" { - return T_DOUBLE_ARROW; + RETURN_TOKEN(T_DOUBLE_ARROW); } <ST_IN_SCRIPTING>"list" { - return T_LIST; + RETURN_TOKEN(T_LIST); } <ST_IN_SCRIPTING>"array" { - return T_ARRAY; + RETURN_TOKEN(T_ARRAY); } <ST_IN_SCRIPTING>"callable" { - return T_CALLABLE; + RETURN_TOKEN(T_CALLABLE); } <ST_IN_SCRIPTING>"++" { - return T_INC; + RETURN_TOKEN(T_INC); } <ST_IN_SCRIPTING>"--" { - return T_DEC; + RETURN_TOKEN(T_DEC); } <ST_IN_SCRIPTING>"===" { - return T_IS_IDENTICAL; + RETURN_TOKEN(T_IS_IDENTICAL); } <ST_IN_SCRIPTING>"!==" { - return T_IS_NOT_IDENTICAL; + RETURN_TOKEN(T_IS_NOT_IDENTICAL); } <ST_IN_SCRIPTING>"==" { - return T_IS_EQUAL; + RETURN_TOKEN(T_IS_EQUAL); } <ST_IN_SCRIPTING>"!="|"<>" { - return T_IS_NOT_EQUAL; + RETURN_TOKEN(T_IS_NOT_EQUAL); } <ST_IN_SCRIPTING>"<=>" { - return T_SPACESHIP; + RETURN_TOKEN(T_SPACESHIP); } <ST_IN_SCRIPTING>"<=" { - return T_IS_SMALLER_OR_EQUAL; + RETURN_TOKEN(T_IS_SMALLER_OR_EQUAL); } <ST_IN_SCRIPTING>">=" { - return T_IS_GREATER_OR_EQUAL; + RETURN_TOKEN(T_IS_GREATER_OR_EQUAL); } <ST_IN_SCRIPTING>"+=" { - return T_PLUS_EQUAL; + RETURN_TOKEN(T_PLUS_EQUAL); } <ST_IN_SCRIPTING>"-=" { - return T_MINUS_EQUAL; + RETURN_TOKEN(T_MINUS_EQUAL); } <ST_IN_SCRIPTING>"*=" { - return T_MUL_EQUAL; + RETURN_TOKEN(T_MUL_EQUAL); } <ST_IN_SCRIPTING>"*\*" { - return T_POW; + RETURN_TOKEN(T_POW); } <ST_IN_SCRIPTING>"*\*=" { - return T_POW_EQUAL; + RETURN_TOKEN(T_POW_EQUAL); } <ST_IN_SCRIPTING>"/=" { - return T_DIV_EQUAL; + RETURN_TOKEN(T_DIV_EQUAL); } <ST_IN_SCRIPTING>".=" { - return T_CONCAT_EQUAL; + RETURN_TOKEN(T_CONCAT_EQUAL); } <ST_IN_SCRIPTING>"%=" { - return T_MOD_EQUAL; + RETURN_TOKEN(T_MOD_EQUAL); } <ST_IN_SCRIPTING>"<<=" { - return T_SL_EQUAL; + RETURN_TOKEN(T_SL_EQUAL); } <ST_IN_SCRIPTING>">>=" { - return T_SR_EQUAL; + RETURN_TOKEN(T_SR_EQUAL); } <ST_IN_SCRIPTING>"&=" { - return T_AND_EQUAL; + RETURN_TOKEN(T_AND_EQUAL); } <ST_IN_SCRIPTING>"|=" { - return T_OR_EQUAL; + RETURN_TOKEN(T_OR_EQUAL); } <ST_IN_SCRIPTING>"^=" { - return T_XOR_EQUAL; + RETURN_TOKEN(T_XOR_EQUAL); } <ST_IN_SCRIPTING>"||" { - return T_BOOLEAN_OR; + RETURN_TOKEN(T_BOOLEAN_OR); } <ST_IN_SCRIPTING>"&&" { - return T_BOOLEAN_AND; + RETURN_TOKEN(T_BOOLEAN_AND); } <ST_IN_SCRIPTING>"OR" { - return T_LOGICAL_OR; + RETURN_TOKEN(T_LOGICAL_OR); } <ST_IN_SCRIPTING>"AND" { - return T_LOGICAL_AND; + RETURN_TOKEN(T_LOGICAL_AND); } <ST_IN_SCRIPTING>"XOR" { - return T_LOGICAL_XOR; + RETURN_TOKEN(T_LOGICAL_XOR); } <ST_IN_SCRIPTING>"<<" { - return T_SL; + RETURN_TOKEN(T_SL); } <ST_IN_SCRIPTING>">>" { - return T_SR; + RETURN_TOKEN(T_SR); } <ST_IN_SCRIPTING>{TOKENS} { - return yytext[0]; + RETURN_TOKEN(yytext[0]); } <ST_IN_SCRIPTING>"{" { yy_push_state(ST_IN_SCRIPTING); - return '{'; + RETURN_TOKEN('{'); } <ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" { yy_push_state(ST_LOOKING_FOR_VARNAME); - return T_DOLLAR_OPEN_CURLY_BRACES; + RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES); } @@ -1578,7 +1601,7 @@ NEWLINE ("\r"|"\n"|"\r\n") if (!zend_stack_is_empty(&SCNG(state_stack))) { yy_pop_state(); } - return '}'; + RETURN_TOKEN('}'); } @@ -1587,7 +1610,7 @@ NEWLINE ("\r"|"\n"|"\r\n") zend_copy_value(zendlval, yytext, yyleng); yy_pop_state(); yy_push_state(ST_IN_SCRIPTING); - return T_STRING_VARNAME; + RETURN_TOKEN(T_STRING_VARNAME); } @@ -1617,12 +1640,12 @@ NEWLINE ("\r"|"\n"|"\r\n") ZVAL_LONG(zendlval, ZEND_STRTOL(bin, &end, 2)); ZEND_ASSERT(!errno && end == yytext + yyleng); } - return T_LNUMBER; + RETURN_TOKEN(T_LNUMBER); } else { ZVAL_DOUBLE(zendlval, zend_bin_strtod(bin, (const char **)&end)); /* errno isn't checked since we allow HUGE_VAL/INF overflow */ ZEND_ASSERT(end == yytext + yyleng); - return T_DNUMBER; + RETURN_TOKEN(T_DNUMBER); } } @@ -1636,7 +1659,7 @@ NEWLINE ("\r"|"\n"|"\r\n") */ if (end != yytext + yyleng) { zend_throw_exception(zend_get_parse_exception(), "Invalid numeric literal", E_PARSE); - return T_ERROR; + RETURN_TOKEN(T_ERROR); } } else { errno = 0; @@ -1653,19 +1676,19 @@ NEWLINE ("\r"|"\n"|"\r\n") if (end != yytext + yyleng) { zend_throw_exception(zend_get_parse_exception(), "Invalid numeric literal", E_PARSE); - return T_ERROR; + RETURN_TOKEN(T_ERROR); } ZEND_ASSERT(!errno); - return T_DNUMBER; + RETURN_TOKEN(T_DNUMBER); } /* Also not an assert for the same reason */ if (end != yytext + yyleng) { zend_throw_exception(zend_get_parse_exception(), "Invalid numeric literal", E_PARSE); - return T_ERROR; + RETURN_TOKEN(T_ERROR); } } ZEND_ASSERT(!errno); - return T_LNUMBER; + RETURN_TOKEN(T_LNUMBER); } <ST_IN_SCRIPTING>{HNUM} { @@ -1687,12 +1710,12 @@ NEWLINE ("\r"|"\n"|"\r\n") ZVAL_LONG(zendlval, ZEND_STRTOL(hex, &end, 16)); ZEND_ASSERT(!errno && end == hex + len); } - return T_LNUMBER; + RETURN_TOKEN(T_LNUMBER); } else { ZVAL_DOUBLE(zendlval, zend_hex_strtod(hex, (const char **)&end)); /* errno isn't checked since we allow HUGE_VAL/INF overflow */ ZEND_ASSERT(end == hex + len); - return T_DNUMBER; + RETURN_TOKEN(T_DNUMBER); } } @@ -1709,12 +1732,12 @@ NEWLINE ("\r"|"\n"|"\r\n") string: ZVAL_STRINGL(zendlval, yytext, yyleng); } - return T_NUM_STRING; + RETURN_TOKEN(T_NUM_STRING); } <ST_VAR_OFFSET>{LNUM}|{HNUM}|{BNUM} { /* Offset must be treated as a string */ ZVAL_STRINGL(zendlval, yytext, yyleng); - return T_NUM_STRING; + RETURN_TOKEN(T_NUM_STRING); } <ST_IN_SCRIPTING>{DNUM}|{EXPONENT_DNUM} { @@ -1723,59 +1746,59 @@ string: ZVAL_DOUBLE(zendlval, zend_strtod(yytext, &end)); /* errno isn't checked since we allow HUGE_VAL/INF overflow */ ZEND_ASSERT(end == yytext + yyleng); - return T_DNUMBER; + RETURN_TOKEN(T_DNUMBER); } <ST_IN_SCRIPTING>"__CLASS__" { - return T_CLASS_C; + RETURN_TOKEN(T_CLASS_C); } <ST_IN_SCRIPTING>"__TRAIT__" { - return T_TRAIT_C; + RETURN_TOKEN(T_TRAIT_C); } <ST_IN_SCRIPTING>"__FUNCTION__" { - return T_FUNC_C; + RETURN_TOKEN(T_FUNC_C); } <ST_IN_SCRIPTING>"__METHOD__" { - return T_METHOD_C; + RETURN_TOKEN(T_METHOD_C); } <ST_IN_SCRIPTING>"__LINE__" { - return T_LINE; + RETURN_TOKEN(T_LINE); } <ST_IN_SCRIPTING>"__FILE__" { - return T_FILE; + RETURN_TOKEN(T_FILE); } <ST_IN_SCRIPTING>"__DIR__" { - return T_DIR; + RETURN_TOKEN(T_DIR); } <ST_IN_SCRIPTING>"__NAMESPACE__" { - return T_NS_C; + RETURN_TOKEN(T_NS_C); } <INITIAL>"<?=" { BEGIN(ST_IN_SCRIPTING); - return T_OPEN_TAG_WITH_ECHO; + RETURN_TOKEN(T_OPEN_TAG_WITH_ECHO); } <INITIAL>"<?php"([ \t]|{NEWLINE}) { HANDLE_NEWLINE(yytext[yyleng-1]); BEGIN(ST_IN_SCRIPTING); - return T_OPEN_TAG; + RETURN_TOKEN(T_OPEN_TAG); } <INITIAL>"<?" { if (CG(short_tags)) { BEGIN(ST_IN_SCRIPTING); - return T_OPEN_TAG; + RETURN_TOKEN(T_OPEN_TAG); } else { goto inline_char_handler; } @@ -1783,7 +1806,7 @@ string: <INITIAL>{ANY_CHAR} { if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } inline_char_handler: @@ -1823,7 +1846,7 @@ inline_char_handler: ZVAL_STRINGL(zendlval, yytext, yyleng); } HANDLE_NEWLINES(yytext, yyleng); - return T_INLINE_HTML; + RETURN_TOKEN(T_INLINE_HTML); } @@ -1834,7 +1857,7 @@ inline_char_handler: yyless(yyleng - 3); yy_push_state(ST_LOOKING_FOR_PROPERTY); zend_copy_value(zendlval, (yytext+1), (yyleng-1)); - return T_VARIABLE; + RETURN_TOKEN(T_VARIABLE); } /* A [ always designates a variable offset, regardless of what follows @@ -1843,22 +1866,22 @@ inline_char_handler: yyless(yyleng - 1); yy_push_state(ST_VAR_OFFSET); zend_copy_value(zendlval, (yytext+1), (yyleng-1)); - return T_VARIABLE; + RETURN_TOKEN(T_VARIABLE); } <ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE,ST_VAR_OFFSET>"$"{LABEL} { zend_copy_value(zendlval, (yytext+1), (yyleng-1)); - return T_VARIABLE; + RETURN_TOKEN(T_VARIABLE); } <ST_VAR_OFFSET>"]" { yy_pop_state(); - return ']'; + RETURN_TOKEN(']'); } <ST_VAR_OFFSET>{TOKENS}|[{}"`] { /* Only '[' can be valid, but returning other tokens will allow a more explicit parse error */ - return yytext[0]; + RETURN_TOKEN(yytext[0]); } <ST_VAR_OFFSET>[ \n\r\t\\'#] { @@ -1866,12 +1889,12 @@ inline_char_handler: yyless(0); yy_pop_state(); ZVAL_NULL(zendlval); - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } <ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} { zend_copy_value(zendlval, yytext, yyleng); - return T_STRING; + RETURN_TOKEN(T_STRING); } @@ -1901,7 +1924,7 @@ inline_char_handler: yyleng = YYCURSOR - SCNG(yy_text); - return T_COMMENT; + RETURN_TOKEN(T_COMMENT); } <ST_IN_SCRIPTING>"/*"|"/**"{WHITESPACE} { @@ -1931,15 +1954,15 @@ inline_char_handler: if (doc_com) { CG(doc_comment) = zend_string_init(yytext, yyleng, 0); - return T_DOC_COMMENT; + RETURN_TOKEN(T_DOC_COMMENT); } - return T_COMMENT; + RETURN_TOKEN(T_COMMENT); } <ST_IN_SCRIPTING>"?>"{NEWLINE}? { BEGIN(INITIAL); - return T_CLOSE_TAG; /* implicit ';' at php-end tag */ + RETURN_TOKEN(T_CLOSE_TAG); /* implicit ';' at php-end tag */ } @@ -1965,7 +1988,7 @@ inline_char_handler: * for ' (unrecognized by parser), instead of old flex fallback to "Unexpected character..." * rule, which continued in ST_IN_SCRIPTING state after the quote */ ZVAL_NULL(zendlval); - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } } @@ -2008,7 +2031,7 @@ inline_char_handler: SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval)); ZVAL_STRINGL(zendlval, str, sz); } - return T_CONSTANT_ENCAPSED_STRING; + RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING); } @@ -2020,9 +2043,9 @@ inline_char_handler: case '"': yyleng = YYCURSOR - SCNG(yy_text); if (zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"') == FAILURE) { - return T_ERROR; + RETURN_TOKEN(T_ERROR); } - return T_CONSTANT_ENCAPSED_STRING; + RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING); case '$': if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') { break; @@ -2052,7 +2075,7 @@ inline_char_handler: YYCURSOR = SCNG(yy_text) + yyleng; BEGIN(ST_DOUBLE_QUOTES); - return '"'; + RETURN_TOKEN('"'); } @@ -2100,13 +2123,13 @@ inline_char_handler: zend_ptr_stack_push(&SCNG(heredoc_label_stack), (void *) heredoc_label); - return T_START_HEREDOC; + RETURN_TOKEN(T_START_HEREDOC); } <ST_IN_SCRIPTING>[`] { BEGIN(ST_BACKQUOTE); - return '`'; + RETURN_TOKEN('`'); } @@ -2120,7 +2143,7 @@ inline_char_handler: efree(heredoc_label); BEGIN(ST_IN_SCRIPTING); - return T_END_HEREDOC; + RETURN_TOKEN(T_END_HEREDOC); } @@ -2128,18 +2151,18 @@ inline_char_handler: Z_LVAL_P(zendlval) = (zend_long) '{'; yy_push_state(ST_IN_SCRIPTING); yyless(1); - return T_CURLY_OPEN; + RETURN_TOKEN(T_CURLY_OPEN); } <ST_DOUBLE_QUOTES>["] { BEGIN(ST_IN_SCRIPTING); - return '"'; + RETURN_TOKEN('"'); } <ST_BACKQUOTE>[`] { BEGIN(ST_IN_SCRIPTING); - return '`'; + RETURN_TOKEN('`'); } @@ -2152,7 +2175,7 @@ inline_char_handler: } if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) { YYCURSOR++; @@ -2189,15 +2212,15 @@ double_quotes_scan_done: yyleng = YYCURSOR - SCNG(yy_text); if (zend_scan_escape_string(zendlval, yytext, yyleng, '"') == FAILURE) { - return T_ERROR; + RETURN_TOKEN(T_ERROR); } - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } <ST_BACKQUOTE>{ANY_CHAR} { if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) { YYCURSOR++; @@ -2233,9 +2256,9 @@ double_quotes_scan_done: yyleng = YYCURSOR - SCNG(yy_text); if (zend_scan_escape_string(zendlval, yytext, yyleng, '`') == FAILURE) { - return T_ERROR; + RETURN_TOKEN(T_ERROR); } - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } @@ -2245,7 +2268,7 @@ double_quotes_scan_done: zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack)); if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } YYCURSOR--; @@ -2309,9 +2332,9 @@ heredoc_scan_done: yyleng = YYCURSOR - SCNG(yy_text); if (zend_scan_escape_string(zendlval, yytext, yyleng - newline, 0) == FAILURE) { - return T_ERROR; + RETURN_TOKEN(T_ERROR); } - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } @@ -2321,7 +2344,7 @@ heredoc_scan_done: zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack)); if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } YYCURSOR--; @@ -2368,13 +2391,13 @@ nowdoc_scan_done: zend_copy_value(zendlval, yytext, yyleng - newline); HANDLE_NEWLINES(yytext, yyleng - newline); - return T_ENCAPSED_AND_WHITESPACE; + RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE); } <ST_IN_SCRIPTING,ST_VAR_OFFSET>{ANY_CHAR} { if (YYCURSOR > YYLIMIT) { - return 0; + RETURN_TOKEN(END); } zend_error(E_COMPILE_WARNING,"Unexpected character in input: '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE); diff --git a/ext/tokenizer/tests/bug67395.phpt b/ext/tokenizer/tests/bug67395.phpt index c9b7f3012f..8101c81edb 100644 --- a/ext/tokenizer/tests/bug67395.phpt +++ b/ext/tokenizer/tests/bug67395.phpt @@ -1,5 +1,7 @@ --TEST-- Bug 67395: token_name() does not return name for T_POW and T_POW_EQUAL token +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> --FILE-- <?php diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt new file mode 100644 index 0000000000..03b991b1a5 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt @@ -0,0 +1,19 @@ +--TEST-- +Parse errors during token_get_all() with TOKEN_PARSE flag +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php + +try { + token_get_all('<?php invalid code;', TOKEN_PARSE); +} catch (ParseException $e) { + echo $e->getMessage(), PHP_EOL; +} + +echo "Done"; + +?> +--EXPECT-- +syntax error, unexpected 'code' (T_STRING) +Done diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt new file mode 100644 index 0000000000..ab334358ab --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt @@ -0,0 +1,81 @@ +--TEST-- +Semi reserved words support: member access +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +$tokens = token_get_all('<?php +X::continue; +X::$continue; +$x->$continue; +X::continue(); +$x->continue(); +X::class; + +class X { + const CONTINUE = 1; + public $x = self::CONTINUE + 1; +} +', TOKEN_PARSE); + +array_walk($tokens, function($tk) { + if(is_array($tk)) { + if(($t = token_name($tk[0])) == 'T_WHITESPACE') return; + echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL; + } + else echo $tk, PHP_EOL; +}); + +echo "Done"; + +?> +--EXPECTF-- +L1: T_OPEN_TAG <?php + +L2: T_STRING X +L2: T_DOUBLE_COLON :: +L2: T_STRING continue +; +L3: T_STRING X +L3: T_DOUBLE_COLON :: +L3: T_VARIABLE $continue +; +L4: T_VARIABLE $x +L4: T_OBJECT_OPERATOR -> +L4: T_VARIABLE $continue +; +L5: T_STRING X +L5: T_DOUBLE_COLON :: +L5: T_STRING continue +( +) +; +L6: T_VARIABLE $x +L6: T_OBJECT_OPERATOR -> +L6: T_STRING continue +( +) +; +L7: T_STRING X +L7: T_DOUBLE_COLON :: +L7: T_CLASS class +; +L9: T_CLASS class +L9: T_STRING X +{ +L10: T_CONST const +L10: T_STRING CONTINUE += +L10: T_LNUMBER 1 +; +L11: T_PUBLIC public +L11: T_VARIABLE $x += +L11: T_STRING self +L11: T_DOUBLE_COLON :: +L11: T_STRING CONTINUE ++ +L11: T_LNUMBER 1 +; +} +Done diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt new file mode 100644 index 0000000000..3dd8e14d84 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt @@ -0,0 +1,68 @@ +--TEST-- +Semi reserved words support: class const +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +$tokens = token_get_all('<?php + class SomeClass { + const CONST = 1; + const CONTINUE = (self::CONST + 1); + const ARRAY = [1, self::CONTINUE => [3, 4], 5]; + } +', TOKEN_PARSE); + +array_walk($tokens, function($tk) { + if(is_array($tk)) { + if(($t = token_name($tk[0])) == 'T_WHITESPACE') return; + echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL; + } + else echo $tk, PHP_EOL; +}); + +echo "Done"; + +?> +--EXPECTF-- +L1: T_OPEN_TAG <?php + +L2: T_CLASS class +L2: T_STRING SomeClass +{ +L3: T_CONST const +L3: T_STRING CONST += +L3: T_LNUMBER 1 +; +L4: T_CONST const +L4: T_STRING CONTINUE += +( +L4: T_STRING self +L4: T_DOUBLE_COLON :: +L4: T_STRING CONST ++ +L4: T_LNUMBER 1 +) +; +L5: T_CONST const +L5: T_STRING ARRAY += +[ +L5: T_LNUMBER 1 +, +L5: T_STRING self +L5: T_DOUBLE_COLON :: +L5: T_STRING CONTINUE +L5: T_DOUBLE_ARROW => +[ +L5: T_LNUMBER 3 +, +L5: T_LNUMBER 4 +] +, +L5: T_LNUMBER 5 +] +; +} +Done diff --git a/ext/tokenizer/tests/token_get_all_error.phpt b/ext/tokenizer/tests/token_get_all_error.phpt index 29e97c38c4..9ded0a1774 100644 --- a/ext/tokenizer/tests/token_get_all_error.phpt +++ b/ext/tokenizer/tests/token_get_all_error.phpt @@ -19,7 +19,7 @@ var_dump( token_get_all()); echo "-- Testing token_get_all() function with more than expected no. of arguments --\n"; $source = '<?php ?>'; $extra_arg = 10; -var_dump( token_get_all($source, $extra_arg)); +var_dump( token_get_all($source, true, $extra_arg)); echo "Done" ?> @@ -28,10 +28,10 @@ echo "Done" -- Testing token_get_all() function with zero arguments -- -Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d +Warning: token_get_all() expects at least 1 parameter, 0 given in %s on line 11 NULL -- Testing token_get_all() function with more than expected no. of arguments -- -Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d +Warning: token_get_all() expects at most 2 parameters, 3 given in %s on line 17 NULL -Done +Done
\ No newline at end of file diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c index c011894441..2a4fa90ca2 100644 --- a/ext/tokenizer/tokenizer.c +++ b/ext/tokenizer/tokenizer.c @@ -37,6 +37,12 @@ #define zendcursor LANG_SCNG(yy_cursor) #define zendlimit LANG_SCNG(yy_limit) +#define TOKEN_PARSE 1 + +void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) { + REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT); +} + /* {{{ arginfo */ ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1) ZEND_ARG_INFO(0, source) @@ -83,6 +89,7 @@ ZEND_GET_MODULE(tokenizer) PHP_MINIT_FUNCTION(tokenizer) { tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU); + tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU); return SUCCESS; } /* }}} */ @@ -97,19 +104,33 @@ PHP_MINFO_FUNCTION(tokenizer) } /* }}} */ -static void tokenize(zval *return_value) +static zend_bool tokenize(zval *return_value, zend_string *source) { + zval source_zval; + zend_lex_state original_lex_state; zval token; zval keyword; int token_type; zend_bool destroy; int token_line = 1; - int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled + int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */ + + ZVAL_STR_COPY(&source_zval, source); + zend_save_lexical_state(&original_lex_state); + if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) { + zend_restore_lexical_state(&original_lex_state); + return 0; + } + + LANG_SCNG(yy_state) = yycINITIAL; array_init(return_value); ZVAL_NULL(&token); while ((token_type = lex_scan(&token))) { + + if(token_type == T_ERROR) break; + destroy = 1; switch (token_type) { case T_CLOSE_TAG: @@ -123,8 +144,6 @@ static void tokenize(zval *return_value) case T_DOC_COMMENT: destroy = 0; break; - case T_ERROR: - return; } if (token_type >= 256) { @@ -147,13 +166,13 @@ static void tokenize(zval *return_value) } ZVAL_NULL(&token); - // after T_HALT_COMPILER collect the next three non-dropped tokens + /* after T_HALT_COMPILER collect the next three non-dropped tokens */ if (need_tokens != -1) { if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG - && token_type != T_COMMENT && token_type != T_DOC_COMMENT - && --need_tokens == 0 + && token_type != T_COMMENT && token_type != T_DOC_COMMENT + && --need_tokens == 0 ) { - // fetch the rest into a T_INLINE_HTML + /* fetch the rest into a T_INLINE_HTML */ if (zendcursor != zendlimit) { array_init(&keyword); add_next_index_long(&keyword, T_INLINE_HTML); @@ -169,34 +188,113 @@ static void tokenize(zval *return_value) token_line = CG(zend_lineno); } + + zval_dtor(&source_zval); + zend_restore_lexical_state(&original_lex_state); + + return 1; } -/* {{{ proto array token_get_all(string source) - */ -PHP_FUNCTION(token_get_all) +zval token_stream; + +void on_event(zend_php_scanner_event event, int token, int line) { - zend_string *source; - zval source_zval; - zend_lex_state original_lex_state; + zval keyword; + HashTable *tokens_ht; + zval *token_zv; - if (zend_parse_parameters(ZEND_NUM_ARGS(), "S", &source) == FAILURE) { - return; + switch(event) { + case ON_TOKEN: + if (token == T_ERROR || token == END) break; + if (token >= 256) { + array_init(&keyword); + add_next_index_long(&keyword, token); + add_next_index_stringl(&keyword, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng)); + add_next_index_long(&keyword, line); + add_next_index_zval(&token_stream, &keyword); + } else { + add_next_index_stringl(&token_stream, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng)); + } + break; + case ON_FEEDBACK: + tokens_ht = Z_ARRVAL(token_stream); + token_zv = zend_hash_index_find(tokens_ht, zend_hash_num_elements(tokens_ht) - 1); + if (token_zv && Z_TYPE_P(token_zv) == IS_ARRAY) { + ZVAL_LONG(zend_hash_index_find(Z_ARRVAL_P(token_zv), 0), token); + } + break; + case ON_STOP: + if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) { + array_init(&keyword); + add_next_index_long(&keyword, T_INLINE_HTML); + add_next_index_stringl(&keyword, + (char *)LANG_SCNG(yy_cursor), LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor)); + add_next_index_long(&keyword, CG(zend_lineno)); + add_next_index_zval(&token_stream, &keyword); + } + break; } +} + +static zend_bool tokenize_parse(zval *return_value, zend_string *source) +{ + zval source_zval; + zend_lex_state original_lex_state; + zend_bool original_in_compilation; + zend_bool success; ZVAL_STR_COPY(&source_zval, source); + + original_in_compilation = CG(in_compilation); + CG(in_compilation) = 1; zend_save_lexical_state(&original_lex_state); - if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) { - zend_restore_lexical_state(&original_lex_state); - RETURN_FALSE; - } + if ((success = (zend_prepare_string_for_scanning(&source_zval, "") == SUCCESS))) { + CG(ast) = NULL; + CG(ast_arena) = zend_arena_create(1024 * 32); + LANG_SCNG(yy_state) = yycINITIAL; + LANG_SCNG(on_event) = on_event; - LANG_SCNG(yy_state) = yycINITIAL; + array_init(&token_stream); + if((success = (zendparse() == SUCCESS))) { + ZVAL_ZVAL(return_value, &token_stream, 1, 0); + } + zval_dtor(&token_stream); - tokenize(return_value); + zend_ast_destroy(CG(ast)); + zend_arena_destroy(CG(ast_arena)); + } + /* restore compiler and scanner global states */ zend_restore_lexical_state(&original_lex_state); + CG(in_compilation) = original_in_compilation; + zval_dtor(&source_zval); + + return success; +} + +/* }}} */ + +/* {{{ proto array token_get_all(string source) + */ +PHP_FUNCTION(token_get_all) +{ + zend_string *source; + zend_long flags = 0; + zend_bool success; + + if (zend_parse_parameters(ZEND_NUM_ARGS(), "S|l", &source, &flags) == FAILURE) { + return; + } + + if (flags & TOKEN_PARSE) { + success = tokenize_parse(return_value, source); + } else { + success = tokenize(return_value, source); + } + + if (!success) RETURN_FALSE; } /* }}} */ diff --git a/tests/basic/bug51709_1.phpt b/tests/basic/bug51709_1.phpt deleted file mode 100644 index 3f2d544e54..0000000000 --- a/tests/basic/bug51709_1.phpt +++ /dev/null @@ -1,16 +0,0 @@ ---TEST-- -Bug #51709 (Can't use keywords as method names) ---FILE-- -<?php - -class foo { - static function for() { - echo "1"; - } -} - -?> -===DONE=== -<?php exit(0); ?> ---EXPECTF-- -Parse error: syntax error, unexpected %s, expecting %s in %sbug51709_1.php on line %d diff --git a/tests/basic/bug51709_2.phpt b/tests/basic/bug51709_2.phpt deleted file mode 100644 index bb1f91cc4c..0000000000 --- a/tests/basic/bug51709_2.phpt +++ /dev/null @@ -1,16 +0,0 @@ ---TEST-- -Bug #51709 (Can't use keywords as method names) ---FILE-- -<?php - -class foo { - static function goto() { - echo "1"; - } -} - -?> -===DONE=== -<?php exit(0); ?> ---EXPECTF-- -Parse error: syntax error, unexpected %s, expecting %s in %sbug51709_2.php on line %d |