diff --git a/src/Fenom/Tokenizer.php b/src/Fenom/Tokenizer.php index 33e8ef1..bc3556d 100644 --- a/src/Fenom/Tokenizer.php +++ b/src/Fenom/Tokenizer.php @@ -136,6 +136,18 @@ class Tokenizer { ) ); + public static $description = array( + self::MACRO_STRING => 'string', + self::MACRO_INCDEC => 'increment/decrement operator', + self::MACRO_UNARY => 'unary operator', + self::MACRO_BINARY => 'binary operator', + self::MACRO_BOOLEAN => 'boolean operator', + self::MACRO_MATH => 'math operator', + self::MACRO_COND => 'conditional operator', + self::MACRO_EQUALS => 'equal operator', + self::MACRO_SCALAR => 'scalar value' + ); + /** * Special tokens * @var array @@ -554,6 +566,18 @@ class Tokenizer { public function getLine() { return $this->curr ? $this->curr[3] : $this->_last_no; } + + /** + * Is current token whitespaced, means previous token has whitespace characters + * @return bool + */ + public function isWhiteSpaced() { + return $this->prev ? (bool)$this->prev[2] : false; + } + + public function getWhitespace() { + return $this->curr ? $this->curr[2] : false; + } } /** @@ -570,6 +594,8 @@ class UnexpectedTokenException extends \RuntimeException { $this->message = "Unexpected end of ".($where?:"expression")."$expect"; } elseif($tokens->curr[0] === T_WHITESPACE) { $this->message = "Unexpected whitespace$expect"; + } elseif($tokens->curr[0] === T_BAD_CHARACTER) { + $this->message = "Unexpected bad characters (below ASCII 32 except \\t, \\n and \\r) in ".($where?:"expression")."$expect"; } else { $this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect"; }