Improve tokenizer

src/Fenom/Tokenizer.php

@@ -136,6 +136,18 @@ class Tokenizer {
         )
     );
 
+    public static $description = array(
+        self::MACRO_STRING  => 'string',
+        self::MACRO_INCDEC  => 'increment/decrement operator',
+        self::MACRO_UNARY   => 'unary operator',
+        self::MACRO_BINARY  => 'binary operator',
+        self::MACRO_BOOLEAN => 'boolean operator',
+        self::MACRO_MATH    => 'math operator',
+        self::MACRO_COND    => 'conditional operator',
+        self::MACRO_EQUALS  => 'equal operator',
+        self::MACRO_SCALAR  => 'scalar value'
+    );
+
     /**
      * Special tokens
      * @var array
@@ -554,6 +566,18 @@ class Tokenizer {
     public function getLine() {
         return $this->curr ? $this->curr[3] : $this->_last_no;
     }
+
+    /**
+     * Is current token whitespaced, means previous token has whitespace characters
+     * @return bool
+     */
+    public function isWhiteSpaced() {
+        return $this->prev ? (bool)$this->prev[2] : false;
+    }
+
+    public function getWhitespace() {
+        return $this->curr ? $this->curr[2] : false;
+    }
 }
 
 /**
@@ -570,6 +594,8 @@ class UnexpectedTokenException extends \RuntimeException {
             $this->message = "Unexpected end of ".($where?:"expression")."$expect";
         } elseif($tokens->curr[0] === T_WHITESPACE) {
             $this->message = "Unexpected whitespace$expect";
+        } elseif($tokens->curr[0] === T_BAD_CHARACTER) {
+            $this->message = "Unexpected bad characters (below ASCII 32 except \\t, \\n and \\r) in ".($where?:"expression")."$expect";
         } else {
             $this->message = "Unexpected token '".$tokens->current()."' in ".($where?:"expression")."$expect";
         }