mirror/fenom
1
0
Fork 0
mirror of https://github.com/fenom-template/fenom.git synced 2023-08-10 21:13:07 +03:00
Code Issues Projects Releases Wiki Activity

Improve tokenizer

Browse Source
This commit is contained in:
bzick 2013-07-20 21:26:02 +04:00
parent f6cc2d5993
commit 46f03c3bd2
1 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/Fenom/Tokenizer.php
View File

@ -136,6 +136,18 @@ class Tokenizer {
)
);
public static $description = array(
self::MACRO_STRING => 'string',
self::MACRO_INCDEC => 'increment/decrement operator',
self::MACRO_UNARY => 'unary operator',
self::MACRO_BINARY => 'binary operator',
self::MACRO_BOOLEAN => 'boolean operator',
self::MACRO_MATH => 'math operator',
self::MACRO_COND => 'conditional operator',
self::MACRO_EQUALS => 'equal operator',
self::MACRO_SCALAR => 'scalar value'
);
/**
* Special tokens
* @var array
@ -554,6 +566,18 @@ class Tokenizer {
public function getLine() {
return $this->curr ? $this->curr[3] : $this->_last_no;
}
/**
* Is current token whitespaced, means previous token has whitespace characters
* @return bool
*/
public function isWhiteSpaced() {
return $this->prev ? (bool)$this->prev[2] : false;
}
public function getWhitespace() {
return $this->curr ? $this->curr[2] : false;
}
}
/**
@ -570,6 +594,8 @@ class UnexpectedTokenException extends \RuntimeException {
$this->message = "Unexpected end of ".($where?:"expression")."$expect";
} elseif($tokens->curr[0] === T_WHITESPACE) {
$this->message = "Unexpected whitespace$expect";
} elseif($tokens->curr[0] === T_BAD_CHARACTER) {
$this->message = "Unexpected bad characters (below ASCII 32 except \\t, \\n and \\r) in ".($where?:"expression")."$expect";
} else {
$this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect";
}