Improve tokenizer

This commit is contained in:
bzick 2013-07-20 21:26:02 +04:00
parent f6cc2d5993
commit 46f03c3bd2

View File

@ -136,6 +136,18 @@ class Tokenizer {
) )
); );
public static $description = array(
self::MACRO_STRING => 'string',
self::MACRO_INCDEC => 'increment/decrement operator',
self::MACRO_UNARY => 'unary operator',
self::MACRO_BINARY => 'binary operator',
self::MACRO_BOOLEAN => 'boolean operator',
self::MACRO_MATH => 'math operator',
self::MACRO_COND => 'conditional operator',
self::MACRO_EQUALS => 'equal operator',
self::MACRO_SCALAR => 'scalar value'
);
/** /**
* Special tokens * Special tokens
* @var array * @var array
@ -554,6 +566,18 @@ class Tokenizer {
public function getLine() { public function getLine() {
return $this->curr ? $this->curr[3] : $this->_last_no; return $this->curr ? $this->curr[3] : $this->_last_no;
} }
/**
* Is current token whitespaced, means previous token has whitespace characters
* @return bool
*/
public function isWhiteSpaced() {
return $this->prev ? (bool)$this->prev[2] : false;
}
public function getWhitespace() {
return $this->curr ? $this->curr[2] : false;
}
} }
/** /**
@ -570,6 +594,8 @@ class UnexpectedTokenException extends \RuntimeException {
$this->message = "Unexpected end of ".($where?:"expression")."$expect"; $this->message = "Unexpected end of ".($where?:"expression")."$expect";
} elseif($tokens->curr[0] === T_WHITESPACE) { } elseif($tokens->curr[0] === T_WHITESPACE) {
$this->message = "Unexpected whitespace$expect"; $this->message = "Unexpected whitespace$expect";
} elseif($tokens->curr[0] === T_BAD_CHARACTER) {
$this->message = "Unexpected bad characters (below ASCII 32 except \\t, \\n and \\r) in ".($where?:"expression")."$expect";
} else { } else {
$this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect"; $this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect";
} }