mirror of
https://github.com/fenom-template/fenom.git
synced 2023-08-10 21:13:07 +03:00
fix Tokenizer
This commit is contained in:
Nikita
parent
dd95df8eb6
commit
cf6bae47cb
@ -12,13 +12,13 @@ namespace Fenom;
|
|||||||
/**
|
/**
|
||||||
* for <PHP 5.4 compatible
|
* for <PHP 5.4 compatible
|
||||||
*/
|
*/
|
||||||
defined('T_INSTEADOF') || define('T_INSTEADOF', 341);
|
defined('T_INSTEADOF') || define('T_INSTEADOF', 341);
|
||||||
defined('T_TRAIT') || define('T_TRAIT', 355);
|
defined('T_TRAIT') || define('T_TRAIT', 355);
|
||||||
defined('T_TRAIT_C') || define('T_TRAIT_C', 365);
|
defined('T_TRAIT_C') || define('T_TRAIT_C', 365);
|
||||||
/**
|
/**
|
||||||
* for PHP <5.5
|
* for PHP <5.5
|
||||||
*/
|
*/
|
||||||
defined('T_YIELD') || define('T_YIELD', 370);
|
defined('T_YIELD') || define('T_YIELD', 370);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Each token have structure
|
* Each token have structure
|
||||||
@ -27,7 +27,7 @@ defined('T_YIELD') || define('T_YIELD', 370);
|
|||||||
* - Whitespace (whitespace symbols after token)
|
* - Whitespace (whitespace symbols after token)
|
||||||
* - Line number of the token
|
* - Line number of the token
|
||||||
*
|
*
|
||||||
* @see http://php.net/tokenizer
|
* @see http://php.net/tokenizer
|
||||||
* @property array $prev the previous token
|
* @property array $prev the previous token
|
||||||
* @property array $curr the current token
|
* @property array $curr the current token
|
||||||
* @property array $next the next token
|
* @property array $next the next token
|
||||||
@ -35,12 +35,12 @@ defined('T_YIELD') || define('T_YIELD', 370);
|
|||||||
* @package Fenom
|
* @package Fenom
|
||||||
* @author Ivan Shalganov <a.cobest@gmail.com>
|
* @author Ivan Shalganov <a.cobest@gmail.com>
|
||||||
*/
|
*/
|
||||||
class Tokenizer {
|
class Tokenizer
|
||||||
const TOKEN = 0;
|
{
|
||||||
const TEXT = 1;
|
const TOKEN = 0;
|
||||||
|
const TEXT = 1;
|
||||||
const WHITESPACE = 2;
|
const WHITESPACE = 2;
|
||||||
const LINE = 3;
|
const LINE = 3;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Some text value: foo, bar, new, class ...
|
* Some text value: foo, bar, new, class ...
|
||||||
*/
|
*/
|
||||||
@ -77,65 +77,57 @@ class Tokenizer {
|
|||||||
* Condition operation
|
* Condition operation
|
||||||
*/
|
*/
|
||||||
const MACRO_COND = 1008;
|
const MACRO_COND = 1008;
|
||||||
|
|
||||||
public $tokens;
|
|
||||||
public $p = 0;
|
|
||||||
public $quotes = 0;
|
|
||||||
private $_max = 0;
|
|
||||||
private $_last_no = 0;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @see http://docs.php.net/manual/en/tokens.php
|
* @see http://docs.php.net/manual/en/tokens.php
|
||||||
* @var array groups of tokens
|
* @var array groups of tokens
|
||||||
*/
|
*/
|
||||||
private static $_macros = array(
|
private static $_macros = array(
|
||||||
self::MACRO_STRING => array(
|
self::MACRO_STRING => array(
|
||||||
\T_ABSTRACT => 1, \T_ARRAY => 1, \T_AS => 1, \T_BREAK => 1, \T_BREAK => 1, \T_CASE => 1,
|
\T_ABSTRACT => 1, \T_ARRAY => 1, \T_AS => 1, \T_BREAK => 1, \T_BREAK => 1, \T_CASE => 1,
|
||||||
\T_CATCH => 1, \T_CLASS => 1, \T_CLASS_C => 1, \T_CLONE => 1, \T_CONST => 1, \T_CONTINUE => 1,
|
\T_CATCH => 1, \T_CLASS => 1, \T_CLASS_C => 1, \T_CLONE => 1, \T_CONST => 1, \T_CONTINUE => 1,
|
||||||
\T_DECLARE => 1, \T_DEFAULT => 1, \T_DIR => 1, \T_DO => 1, \T_ECHO => 1, \T_ELSE => 1,
|
\T_DECLARE => 1, \T_DEFAULT => 1, \T_DIR => 1, \T_DO => 1, \T_ECHO => 1, \T_ELSE => 1,
|
||||||
\T_ELSEIF => 1, \T_EMPTY => 1, \T_ENDDECLARE => 1, \T_ENDFOR => 1, \T_ENDFOREACH => 1, \T_ENDIF => 1,
|
\T_ELSEIF => 1, \T_EMPTY => 1, \T_ENDDECLARE => 1, \T_ENDFOR => 1, \T_ENDFOREACH => 1, \T_ENDIF => 1,
|
||||||
\T_ENDSWITCH => 1, \T_ENDWHILE => 1, \T_EVAL => 1, \T_EXIT => 1, \T_EXTENDS => 1, \T_FILE => 1,
|
\T_ENDSWITCH => 1, \T_ENDWHILE => 1, \T_EVAL => 1, \T_EXIT => 1, \T_EXTENDS => 1, \T_FILE => 1,
|
||||||
\T_FINAL => 1, \T_FOR => 1, \T_FOREACH => 1, \T_FUNCTION => 1, \T_FUNC_C => 1, \T_GLOBAL => 1,
|
\T_FINAL => 1, \T_FOR => 1, \T_FOREACH => 1, \T_FUNCTION => 1, \T_FUNC_C => 1, \T_GLOBAL => 1,
|
||||||
\T_GOTO => 1, \T_HALT_COMPILER => 1, \T_IF => 1, \T_IMPLEMENTS => 1, \T_INCLUDE => 1, \T_INCLUDE_ONCE => 1,
|
\T_GOTO => 1, \T_HALT_COMPILER => 1, \T_IF => 1, \T_IMPLEMENTS => 1, \T_INCLUDE => 1, \T_INCLUDE_ONCE => 1,
|
||||||
\T_INSTANCEOF => 1, \T_INSTEADOF => 1, \T_INTERFACE => 1, \T_ISSET => 1, \T_LINE => 1, \T_LIST => 1,
|
\T_INSTANCEOF => 1, \T_INSTEADOF => 1, \T_INTERFACE => 1, \T_ISSET => 1, \T_LINE => 1, \T_LIST => 1,
|
||||||
\T_LOGICAL_AND => 1, \T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_METHOD_C => 1, \T_NAMESPACE => 1, \T_NS_C => 1,
|
\T_LOGICAL_AND => 1, \T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_METHOD_C => 1, \T_NAMESPACE => 1, \T_NS_C => 1,
|
||||||
\T_NEW => 1, \T_PRINT => 1, \T_PRIVATE => 1, \T_PUBLIC => 1, \T_PROTECTED => 1, \T_REQUIRE => 1,
|
\T_NEW => 1, \T_PRINT => 1, \T_PRIVATE => 1, \T_PUBLIC => 1, \T_PROTECTED => 1, \T_REQUIRE => 1,
|
||||||
\T_REQUIRE_ONCE => 1,\T_RETURN => 1, \T_RETURN => 1, \T_STRING => 1, \T_SWITCH => 1, \T_THROW => 1,
|
\T_REQUIRE_ONCE => 1, \T_RETURN => 1, \T_RETURN => 1, \T_STRING => 1, \T_SWITCH => 1, \T_THROW => 1,
|
||||||
\T_TRAIT => 1, \T_TRAIT_C => 1, \T_TRY => 1, \T_UNSET => 1, \T_UNSET => 1, \T_VAR => 1,
|
\T_TRAIT => 1, \T_TRAIT_C => 1, \T_TRY => 1, \T_UNSET => 1, \T_UNSET => 1, \T_VAR => 1,
|
||||||
\T_WHILE => 1, \T_YIELD => 1, \T_USE => 1
|
\T_WHILE => 1, \T_YIELD => 1, \T_USE => 1
|
||||||
),
|
),
|
||||||
self::MACRO_INCDEC => array(
|
self::MACRO_INCDEC => array(
|
||||||
\T_INC => 1, \T_DEC => 1
|
\T_INC => 1, \T_DEC => 1
|
||||||
),
|
),
|
||||||
self::MACRO_UNARY => array(
|
self::MACRO_UNARY => array(
|
||||||
"!" => 1, "~" => 1, "-" => 1
|
"!" => 1, "~" => 1, "-" => 1
|
||||||
),
|
),
|
||||||
self::MACRO_BINARY => array(
|
self::MACRO_BINARY => array(
|
||||||
\T_BOOLEAN_AND => 1, \T_BOOLEAN_OR => 1, \T_IS_GREATER_OR_EQUAL => 1, \T_IS_EQUAL => 1, \T_IS_IDENTICAL => 1,
|
\T_BOOLEAN_AND => 1, \T_BOOLEAN_OR => 1, \T_IS_GREATER_OR_EQUAL => 1, \T_IS_EQUAL => 1, \T_IS_IDENTICAL => 1,
|
||||||
\T_IS_NOT_EQUAL => 1,\T_IS_NOT_IDENTICAL => 1, \T_IS_SMALLER_OR_EQUAL => 1, \T_LOGICAL_AND => 1,
|
\T_IS_NOT_EQUAL => 1, \T_IS_NOT_IDENTICAL => 1, \T_IS_SMALLER_OR_EQUAL => 1, \T_LOGICAL_AND => 1,
|
||||||
\T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_SL => 1, \T_SR => 1,
|
\T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_SL => 1, \T_SR => 1,
|
||||||
"+" => 1, "-" => 1, "*" => 1, "/" => 1, ">" => 1, "<" => 1, "^" => 1, "%" => 1, "&" => 1
|
"+" => 1, "-" => 1, "*" => 1, "/" => 1, ">" => 1, "<" => 1, "^" => 1, "%" => 1, "&" => 1
|
||||||
),
|
),
|
||||||
self::MACRO_BOOLEAN => array(
|
self::MACRO_BOOLEAN => array(
|
||||||
\T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_BOOLEAN_AND => 1, \T_BOOLEAN_OR => 1, \T_LOGICAL_AND => 1
|
\T_LOGICAL_OR => 1, \T_LOGICAL_XOR => 1, \T_BOOLEAN_AND => 1, \T_BOOLEAN_OR => 1, \T_LOGICAL_AND => 1
|
||||||
),
|
),
|
||||||
self::MACRO_MATH => array(
|
self::MACRO_MATH => array(
|
||||||
"+" => 1, "-" => 1, "*" => 1, "/" => 1, "^" => 1, "%" => 1, "&" => 1, "|" => 1
|
"+" => 1, "-" => 1, "*" => 1, "/" => 1, "^" => 1, "%" => 1, "&" => 1, "|" => 1
|
||||||
),
|
),
|
||||||
self::MACRO_COND => array(
|
self::MACRO_COND => array(
|
||||||
\T_IS_EQUAL => 1, \T_IS_IDENTICAL => 1, ">" => 1, "<" => 1, \T_SL => 1, \T_SR => 1,
|
\T_IS_EQUAL => 1, \T_IS_IDENTICAL => 1, ">" => 1, "<" => 1, \T_SL => 1, \T_SR => 1,
|
||||||
\T_IS_NOT_EQUAL => 1,\T_IS_NOT_IDENTICAL => 1, \T_IS_SMALLER_OR_EQUAL => 1,
|
\T_IS_NOT_EQUAL => 1, \T_IS_NOT_IDENTICAL => 1, \T_IS_SMALLER_OR_EQUAL => 1,
|
||||||
),
|
),
|
||||||
self::MACRO_EQUALS => array(
|
self::MACRO_EQUALS => array(
|
||||||
\T_AND_EQUAL => 1, \T_CONCAT_EQUAL => 1,\T_DIV_EQUAL => 1, \T_MINUS_EQUAL => 1, \T_MOD_EQUAL => 1,
|
\T_AND_EQUAL => 1, \T_CONCAT_EQUAL => 1, \T_DIV_EQUAL => 1, \T_MINUS_EQUAL => 1, \T_MOD_EQUAL => 1,
|
||||||
\T_MUL_EQUAL => 1, \T_OR_EQUAL => 1, \T_PLUS_EQUAL => 1, \T_SL_EQUAL => 1, \T_SR_EQUAL => 1,
|
\T_MUL_EQUAL => 1, \T_OR_EQUAL => 1, \T_PLUS_EQUAL => 1, \T_SL_EQUAL => 1, \T_SR_EQUAL => 1,
|
||||||
\T_XOR_EQUAL => 1, '=' => 1
|
\T_XOR_EQUAL => 1, '=' => 1
|
||||||
),
|
),
|
||||||
self::MACRO_SCALAR => array(
|
self::MACRO_SCALAR => array(
|
||||||
\T_LNUMBER => 1, \T_DNUMBER => 1, \T_CONSTANT_ENCAPSED_STRING => 1
|
\T_LNUMBER => 1, \T_DNUMBER => 1, \T_CONSTANT_ENCAPSED_STRING => 1
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Special tokens
|
* Special tokens
|
||||||
* @var array
|
* @var array
|
||||||
@ -143,19 +135,25 @@ class Tokenizer {
|
|||||||
private static $spec = array(
|
private static $spec = array(
|
||||||
'true' => 1, 'false' => 1, 'null' => 1, 'TRUE' => 1, 'FALSE' => 1, 'NULL' => 1
|
'true' => 1, 'false' => 1, 'null' => 1, 'TRUE' => 1, 'FALSE' => 1, 'NULL' => 1
|
||||||
);
|
);
|
||||||
|
public $tokens;
|
||||||
|
public $p = 0;
|
||||||
|
public $quotes = 0;
|
||||||
|
private $_max = 0;
|
||||||
|
private $_last_no = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param $query
|
* @param $query
|
||||||
*/
|
*/
|
||||||
public function __construct($query) {
|
public function __construct($query)
|
||||||
$tokens = array(-1 => array(\T_WHITESPACE, '', '', 1));
|
{
|
||||||
$_tokens = token_get_all("<?php ".$query);
|
$tokens = array(-1 => array(\T_WHITESPACE, '', '', 1));
|
||||||
$line = 1;
|
$_tokens = token_get_all("<?php " . $query);
|
||||||
|
$line = 1;
|
||||||
array_shift($_tokens);
|
array_shift($_tokens);
|
||||||
$i = 0;
|
$i = 0;
|
||||||
foreach($_tokens as $token) {
|
foreach ($_tokens as $token) {
|
||||||
if(is_string($token)) {
|
if (is_string($token)) {
|
||||||
if($token === '"' || $token === "'" || $token === "`") {
|
if ($token === '"' || $token === "'" || $token === "`") {
|
||||||
$this->quotes++;
|
$this->quotes++;
|
||||||
}
|
}
|
||||||
$tokens[] = array(
|
$tokens[] = array(
|
||||||
@ -166,13 +164,13 @@ class Tokenizer {
|
|||||||
);
|
);
|
||||||
$i++;
|
$i++;
|
||||||
} elseif ($token[0] === \T_WHITESPACE) {
|
} elseif ($token[0] === \T_WHITESPACE) {
|
||||||
$tokens[$i-1][2] = $token[1];
|
$tokens[$i - 1][2] = $token[1];
|
||||||
} else {
|
} else {
|
||||||
$tokens[] = array(
|
$tokens[] = array(
|
||||||
$token[0],
|
$token[0],
|
||||||
$token[1],
|
$token[1],
|
||||||
"",
|
"",
|
||||||
$line = $token[2],
|
$line = $token[2],
|
||||||
token_name($token[0]) // debug
|
token_name($token[0]) // debug
|
||||||
);
|
);
|
||||||
$i++;
|
$i++;
|
||||||
@ -180,17 +178,37 @@ class Tokenizer {
|
|||||||
|
|
||||||
}
|
}
|
||||||
unset($tokens[-1]);
|
unset($tokens[-1]);
|
||||||
$this->tokens = $tokens;
|
$this->tokens = $tokens;
|
||||||
$this->_max = count($this->tokens) - 1;
|
$this->_max = count($this->tokens) - 1;
|
||||||
$this->_last_no = $this->tokens[$this->_max][3];
|
$this->_last_no = $this->tokens[$this->_max][3];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get token name
|
||||||
|
* @static
|
||||||
|
* @param int|string $token
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public static function getName($token)
|
||||||
|
{
|
||||||
|
if (is_string($token)) {
|
||||||
|
return $token;
|
||||||
|
} elseif (is_integer($token)) {
|
||||||
|
return token_name($token);
|
||||||
|
} elseif (is_array($token)) {
|
||||||
|
return token_name($token[0]);
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Is incomplete mean some string not closed
|
* Is incomplete mean some string not closed
|
||||||
*
|
*
|
||||||
* @return int
|
* @return int
|
||||||
*/
|
*/
|
||||||
public function isIncomplete() {
|
public function isIncomplete()
|
||||||
|
{
|
||||||
return ($this->quotes % 2) || ($this->tokens[$this->_max][0] === T_ENCAPSED_AND_WHITESPACE);
|
return ($this->quotes % 2) || ($this->tokens[$this->_max][0] === T_ENCAPSED_AND_WHITESPACE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -200,7 +218,8 @@ class Tokenizer {
|
|||||||
* @link http://php.net/manual/en/iterator.current.php
|
* @link http://php.net/manual/en/iterator.current.php
|
||||||
* @return mixed Can return any type.
|
* @return mixed Can return any type.
|
||||||
*/
|
*/
|
||||||
public function current() {
|
public function current()
|
||||||
|
{
|
||||||
return $this->curr[1];
|
return $this->curr[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -210,51 +229,31 @@ class Tokenizer {
|
|||||||
* @link http://php.net/manual/en/iterator.next.php
|
* @link http://php.net/manual/en/iterator.next.php
|
||||||
* @return Tokenizer
|
* @return Tokenizer
|
||||||
*/
|
*/
|
||||||
public function next() {
|
public function next()
|
||||||
if($this->p > $this->_max) {
|
{
|
||||||
|
if ($this->p > $this->_max) {
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
$this->p++;
|
$this->p++;
|
||||||
unset($this->prev, $this->curr, $this->next);
|
unset($this->prev, $this->curr, $this->next);
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Check token type. If token type is one of expected types return true. Otherwise return false
|
|
||||||
*
|
|
||||||
* @param array $expects
|
|
||||||
* @param string|int $token
|
|
||||||
* @return bool
|
|
||||||
*/
|
|
||||||
private function _valid($expects, $token) {
|
|
||||||
foreach($expects as $expect) {
|
|
||||||
if(is_string($expect) || $expect < 1000) {
|
|
||||||
if($expect === $token) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
|
|
||||||
if(isset(self::$_macros[ $expect ][ $token ])) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* If the next token is a valid one, move the position of cursor one step forward. Otherwise throws an exception.
|
* If the next token is a valid one, move the position of cursor one step forward. Otherwise throws an exception.
|
||||||
* @param array $tokens
|
* @param array $tokens
|
||||||
* @throws UnexpectedTokenException
|
* @throws UnexpectedTokenException
|
||||||
* @return mixed
|
* @return mixed
|
||||||
*/
|
*/
|
||||||
public function _next($tokens) {
|
public function _next($tokens)
|
||||||
|
{
|
||||||
$this->next();
|
$this->next();
|
||||||
if(!$this->curr) {
|
if (!$this->curr) {
|
||||||
throw new UnexpectedTokenException($this, $tokens);
|
throw new UnexpectedTokenException($this, $tokens);
|
||||||
}
|
}
|
||||||
if($tokens) {
|
if ($tokens) {
|
||||||
if($this->_valid($tokens, $this->key())) {
|
if ($this->_valid($tokens, $this->key())) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@ -267,28 +266,31 @@ class Tokenizer {
|
|||||||
* Fetch next specified token or throw an exception
|
* Fetch next specified token or throw an exception
|
||||||
* @return mixed
|
* @return mixed
|
||||||
*/
|
*/
|
||||||
public function getNext(/*int|string $token1, int|string $token2, ... */) {
|
public function getNext( /*int|string $token1, int|string $token2, ... */)
|
||||||
|
{
|
||||||
$this->_next(func_get_args());
|
$this->_next(func_get_args());
|
||||||
|
|
||||||
return $this->current();
|
return $this->current();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return substring. This method doesn't move pointer.
|
* Return substring. This method doesn't move pointer.
|
||||||
* @param int $offset
|
* @param int $offset
|
||||||
* @param int $limit
|
* @param int $limit
|
||||||
* @return string
|
* @return string
|
||||||
*/
|
*/
|
||||||
public function getSubstr($offset, $limit = 0) {
|
public function getSubstr($offset, $limit = 0)
|
||||||
|
{
|
||||||
$str = '';
|
$str = '';
|
||||||
if(!$limit) {
|
if (!$limit) {
|
||||||
$limit = $this->_max;
|
$limit = $this->_max;
|
||||||
} else {
|
} else {
|
||||||
$limit += $offset;
|
$limit += $offset;
|
||||||
}
|
}
|
||||||
for($i = $offset; $i <= $limit; $i++){
|
for ($i = $offset; $i <= $limit; $i++) {
|
||||||
$str .= $this->tokens[$i][1].$this->tokens[$i][2];
|
$str .= $this->tokens[$i][1] . $this->tokens[$i][2];
|
||||||
}
|
}
|
||||||
|
|
||||||
return $str;
|
return $str;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -297,10 +299,12 @@ class Tokenizer {
|
|||||||
* @return mixed
|
* @return mixed
|
||||||
* @throws UnexpectedTokenException
|
* @throws UnexpectedTokenException
|
||||||
*/
|
*/
|
||||||
public function getAndNext() {
|
public function getAndNext()
|
||||||
if($this->curr) {
|
{
|
||||||
|
if ($this->curr) {
|
||||||
$cur = $this->curr[1];
|
$cur = $this->curr[1];
|
||||||
$this->next();
|
$this->next();
|
||||||
|
|
||||||
return $cur;
|
return $cur;
|
||||||
} else {
|
} else {
|
||||||
throw new UnexpectedTokenException($this, func_get_args());
|
throw new UnexpectedTokenException($this, func_get_args());
|
||||||
@ -312,7 +316,8 @@ class Tokenizer {
|
|||||||
* @param $token1
|
* @param $token1
|
||||||
* @return bool
|
* @return bool
|
||||||
*/
|
*/
|
||||||
public function isNext($token1/*, ...*/) {
|
public function isNext($token1 /*, ...*/)
|
||||||
|
{
|
||||||
return $this->next && $this->_valid(func_get_args(), $this->next[0]);
|
return $this->next && $this->_valid(func_get_args(), $this->next[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -321,7 +326,8 @@ class Tokenizer {
|
|||||||
* @param $token1
|
* @param $token1
|
||||||
* @return bool
|
* @return bool
|
||||||
*/
|
*/
|
||||||
public function is($token1/*, ...*/) {
|
public function is($token1 /*, ...*/)
|
||||||
|
{
|
||||||
return $this->curr && $this->_valid(func_get_args(), $this->curr[0]);
|
return $this->curr && $this->_valid(func_get_args(), $this->curr[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -330,7 +336,8 @@ class Tokenizer {
|
|||||||
* @param $token1
|
* @param $token1
|
||||||
* @return bool
|
* @return bool
|
||||||
*/
|
*/
|
||||||
public function isPrev($token1/*, ...*/) {
|
public function isPrev($token1 /*, ...*/)
|
||||||
|
{
|
||||||
return $this->prev && $this->_valid(func_get_args(), $this->prev[0]);
|
return $this->prev && $this->_valid(func_get_args(), $this->prev[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -341,8 +348,9 @@ class Tokenizer {
|
|||||||
* @throws UnexpectedTokenException
|
* @throws UnexpectedTokenException
|
||||||
* @return mixed
|
* @return mixed
|
||||||
*/
|
*/
|
||||||
public function get($token1 /*, $token2 ...*/) {
|
public function get($token1 /*, $token2 ...*/)
|
||||||
if($this->curr && $this->_valid(func_get_args(), $this->curr[0])) {
|
{
|
||||||
|
if ($this->curr && $this->_valid(func_get_args(), $this->curr[0])) {
|
||||||
return $this->curr[1];
|
return $this->curr[1];
|
||||||
} else {
|
} else {
|
||||||
throw new UnexpectedTokenException($this, func_get_args());
|
throw new UnexpectedTokenException($this, func_get_args());
|
||||||
@ -353,12 +361,14 @@ class Tokenizer {
|
|||||||
* Step back
|
* Step back
|
||||||
* @return Tokenizer
|
* @return Tokenizer
|
||||||
*/
|
*/
|
||||||
public function back() {
|
public function back()
|
||||||
if($this->p === 0) {
|
{
|
||||||
|
if ($this->p === 0) {
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
$this->p--;
|
$this->p--;
|
||||||
unset($this->prev, $this->curr, $this->next);
|
unset($this->prev, $this->curr, $this->next);
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -368,8 +378,9 @@ class Tokenizer {
|
|||||||
* @param string $key
|
* @param string $key
|
||||||
* @return mixed
|
* @return mixed
|
||||||
*/
|
*/
|
||||||
public function __get($key) {
|
public function __get($key)
|
||||||
switch($key) {
|
{
|
||||||
|
switch ($key) {
|
||||||
case 'curr':
|
case 'curr':
|
||||||
return $this->curr = ($this->p <= $this->_max) ? $this->tokens[$this->p] : null;
|
return $this->curr = ($this->p <= $this->_max) ? $this->tokens[$this->p] : null;
|
||||||
case 'next':
|
case 'next':
|
||||||
@ -381,7 +392,8 @@ class Tokenizer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function count() {
|
public function count()
|
||||||
|
{
|
||||||
return $this->_max;
|
return $this->_max;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -389,7 +401,8 @@ class Tokenizer {
|
|||||||
* Return the key of the current element
|
* Return the key of the current element
|
||||||
* @return mixed scalar on success, or null on failure.
|
* @return mixed scalar on success, or null on failure.
|
||||||
*/
|
*/
|
||||||
public function key() {
|
public function key()
|
||||||
|
{
|
||||||
return $this->curr ? $this->curr[0] : null;
|
return $this->curr ? $this->curr[0] : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -398,44 +411,30 @@ class Tokenizer {
|
|||||||
* @return boolean The return value will be casted to boolean and then evaluated.
|
* @return boolean The return value will be casted to boolean and then evaluated.
|
||||||
* Returns true on success or false on failure.
|
* Returns true on success or false on failure.
|
||||||
*/
|
*/
|
||||||
public function valid() {
|
public function valid()
|
||||||
|
{
|
||||||
return (bool)$this->curr;
|
return (bool)$this->curr;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get token name
|
|
||||||
* @static
|
|
||||||
* @param int|string $token
|
|
||||||
* @return string
|
|
||||||
*/
|
|
||||||
public static function getName($token) {
|
|
||||||
if(is_string($token)) {
|
|
||||||
return $token;
|
|
||||||
} elseif(is_integer($token)) {
|
|
||||||
return token_name($token);
|
|
||||||
} elseif(is_array($token)) {
|
|
||||||
return token_name($token[0]);
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Skip specific token or throw an exception
|
* Skip specific token or throw an exception
|
||||||
*
|
*
|
||||||
* @throws UnexpectedTokenException
|
* @throws UnexpectedTokenException
|
||||||
* @return Tokenizer
|
* @return Tokenizer
|
||||||
*/
|
*/
|
||||||
public function skip(/*$token1, $token2, ...*/) {
|
public function skip( /*$token1, $token2, ...*/)
|
||||||
if(func_num_args()) {
|
{
|
||||||
if($this->_valid(func_get_args(), $this->curr[0])) {
|
if (func_num_args()) {
|
||||||
|
if ($this->_valid(func_get_args(), $this->curr[0])) {
|
||||||
$this->next();
|
$this->next();
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
} else {
|
} else {
|
||||||
throw new UnexpectedTokenException($this, func_get_args());
|
throw new UnexpectedTokenException($this, func_get_args());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
$this->next();
|
$this->next();
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -446,10 +445,12 @@ class Tokenizer {
|
|||||||
* @param int|string $token1
|
* @param int|string $token1
|
||||||
* @return Tokenizer
|
* @return Tokenizer
|
||||||
*/
|
*/
|
||||||
public function skipIf($token1/*, $token2, ...*/) {
|
public function skipIf($token1 /*, $token2, ...*/)
|
||||||
if($this->_valid(func_get_args(), $this->curr[0])) {
|
{
|
||||||
|
if ($this->_valid(func_get_args(), $this->curr[0])) {
|
||||||
$this->next();
|
$this->next();
|
||||||
}
|
}
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -460,8 +461,9 @@ class Tokenizer {
|
|||||||
* @return Tokenizer
|
* @return Tokenizer
|
||||||
* @throws UnexpectedTokenException
|
* @throws UnexpectedTokenException
|
||||||
*/
|
*/
|
||||||
public function need($token1/*, $token2, ...*/) {
|
public function need($token1 /*, $token2, ...*/)
|
||||||
if($this->_valid(func_get_args(), $this->curr[0])) {
|
{
|
||||||
|
if ($this->_valid(func_get_args(), $this->curr[0])) {
|
||||||
return $this;
|
return $this;
|
||||||
} else {
|
} else {
|
||||||
throw new UnexpectedTokenException($this, func_get_args());
|
throw new UnexpectedTokenException($this, func_get_args());
|
||||||
@ -471,40 +473,41 @@ class Tokenizer {
|
|||||||
/**
|
/**
|
||||||
* Get tokens near current position
|
* Get tokens near current position
|
||||||
* @param int $before count tokens before current token
|
* @param int $before count tokens before current token
|
||||||
* @param int $after count tokens after current token
|
* @param int $after count tokens after current token
|
||||||
* @return array
|
* @return array
|
||||||
*/
|
*/
|
||||||
public function getSnippet($before = 0, $after = 0) {
|
public function getSnippet($before = 0, $after = 0)
|
||||||
|
{
|
||||||
$from = 0;
|
$from = 0;
|
||||||
$to = $this->p;
|
$to = $this->p;
|
||||||
if($before > 0) {
|
if ($before > 0) {
|
||||||
if($before > $this->p) {
|
if ($before > $this->p) {
|
||||||
$from = $this->p;
|
$from = $this->p;
|
||||||
} else {
|
} else {
|
||||||
$from = $before;
|
$from = $before;
|
||||||
}
|
}
|
||||||
} elseif($before < 0) {
|
} elseif ($before < 0) {
|
||||||
$from = $this->p + $before;
|
$from = $this->p + $before;
|
||||||
if($from < 0) {
|
if ($from < 0) {
|
||||||
$from = 0;
|
$from = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if($after > 0) {
|
if ($after > 0) {
|
||||||
$to = $this->p + $after;
|
$to = $this->p + $after;
|
||||||
if($to > $this->_max) {
|
if ($to > $this->_max) {
|
||||||
$to = $this->_max;
|
$to = $this->_max;
|
||||||
}
|
}
|
||||||
} elseif($after < 0) {
|
} elseif ($after < 0) {
|
||||||
$to = $this->_max + $after;
|
$to = $this->_max + $after;
|
||||||
if($to < $this->p) {
|
if ($to < $this->p) {
|
||||||
$to = $this->p;
|
$to = $this->p;
|
||||||
}
|
}
|
||||||
} elseif($this->p > $this->_max) {
|
} elseif ($this->p > $this->_max) {
|
||||||
$to = $this->_max;
|
$to = $this->_max;
|
||||||
}
|
}
|
||||||
$code = array();
|
$code = array();
|
||||||
for($i=$from; $i<=$to; $i++) {
|
for ($i = $from; $i <= $to; $i++) {
|
||||||
$code[] = $this->tokens[ $i ];
|
$code[] = $this->tokens[$i];
|
||||||
}
|
}
|
||||||
|
|
||||||
return $code;
|
return $code;
|
||||||
@ -516,11 +519,13 @@ class Tokenizer {
|
|||||||
* @param int $after
|
* @param int $after
|
||||||
* @return string
|
* @return string
|
||||||
*/
|
*/
|
||||||
public function getSnippetAsString($before = 0, $after = 0) {
|
public function getSnippetAsString($before = 0, $after = 0)
|
||||||
|
{
|
||||||
$str = "";
|
$str = "";
|
||||||
foreach($this->getSnippet($before, $after) as $token) {
|
foreach ($this->getSnippet($before, $after) as $token) {
|
||||||
$str .= $token[1].$token[2];
|
$str .= $token[1] . $token[2];
|
||||||
}
|
}
|
||||||
|
|
||||||
return trim(str_replace("\n", '↵', $str));
|
return trim(str_replace("\n", '↵', $str));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -528,7 +533,8 @@ class Tokenizer {
|
|||||||
* Check if current is special value: true, TRUE, false, FALSE, null, NULL
|
* Check if current is special value: true, TRUE, false, FALSE, null, NULL
|
||||||
* @return bool
|
* @return bool
|
||||||
*/
|
*/
|
||||||
public function isSpecialVal() {
|
public function isSpecialVal()
|
||||||
|
{
|
||||||
return isset(self::$spec[$this->current()]);
|
return isset(self::$spec[$this->current()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -536,14 +542,16 @@ class Tokenizer {
|
|||||||
* Check if the token is last
|
* Check if the token is last
|
||||||
* @return bool
|
* @return bool
|
||||||
*/
|
*/
|
||||||
public function isLast() {
|
public function isLast()
|
||||||
|
{
|
||||||
return $this->p === $this->_max;
|
return $this->p === $this->_max;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Move pointer to the end
|
* Move pointer to the end
|
||||||
*/
|
*/
|
||||||
public function end() {
|
public function end()
|
||||||
|
{
|
||||||
$this->p = $this->_max;
|
$this->p = $this->_max;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -551,27 +559,57 @@ class Tokenizer {
|
|||||||
* Return line number of the current token
|
* Return line number of the current token
|
||||||
* @return mixed
|
* @return mixed
|
||||||
*/
|
*/
|
||||||
public function getLine() {
|
public function getLine()
|
||||||
|
{
|
||||||
return $this->curr ? $this->curr[3] : $this->_last_no;
|
return $this->curr ? $this->curr[3] : $this->_last_no;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check token type. If token type is one of expected types return true. Otherwise return false
|
||||||
|
*
|
||||||
|
* @param array $expects
|
||||||
|
* @param string|int $token
|
||||||
|
* @return bool
|
||||||
|
*/
|
||||||
|
private function _valid($expects, $token)
|
||||||
|
{
|
||||||
|
foreach ($expects as $expect) {
|
||||||
|
if (is_string($expect) || $expect < 1000) {
|
||||||
|
if ($expect === $token) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
|
||||||
|
if (isset(self::$_macros[$expect][$token])) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unexpected token
|
* Unexpected token
|
||||||
*/
|
*/
|
||||||
class UnexpectedTokenException extends \RuntimeException {
|
class UnexpectedTokenException extends \RuntimeException
|
||||||
public function __construct(Tokenizer $tokens, $expect = null, $where = null) {
|
{
|
||||||
if($expect && count($expect) == 1 && is_string($expect[0])) {
|
public function __construct(Tokenizer $tokens, $expect = null, $where = null)
|
||||||
$expect = ", expect '".$expect[0]."'";
|
{
|
||||||
|
if ($expect && count($expect) == 1 && is_string($expect[0])) {
|
||||||
|
$expect = ", expect '" . $expect[0] . "'";
|
||||||
} else {
|
} else {
|
||||||
$expect = "";
|
$expect = "";
|
||||||
}
|
}
|
||||||
if(!$tokens->curr) {
|
if (!$tokens->curr) {
|
||||||
$this->message = "Unexpected end of ".($where?:"expression")."$expect";
|
$this->message = "Unexpected end of " . ($where ? : "expression") . "$expect";
|
||||||
} elseif($tokens->curr[0] === T_WHITESPACE) {
|
} elseif ($tokens->curr[0] === T_WHITESPACE) {
|
||||||
$this->message = "Unexpected whitespace$expect";
|
$this->message = "Unexpected whitespace$expect";
|
||||||
} else {
|
} else {
|
||||||
$this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect";
|
$this->message = "Unexpected token '" . $tokens->current() . "' in " . ($where ? : "expression") . "$expect";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
|
;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user