Fix: reparse close tag, custom provider, simplify compile method, add providers and tests

benchmark/templates/foreach/smarty.tpl

@@ -1,4 +1,4 @@
-<h1>Вывод 10 полей из 1000 элементов в цикле<h1>
+<h1>Вывод 10 полей из 1000 элементов в цикле</h1>
 {foreach $array as $item}
 {$item.id} {$item.title} {$item.var1} {$item.var2} {$item.var3} {$item.var4} {$item.var5} {$item.var6} {$item.var5} {$item.var6} 
-{/foreach}
+{/foreach}

docs/main.md

@@ -12,6 +12,8 @@ Documentation
 
 ### Modifiers
 
+[Usage](./syntax.md#modifiers)
+
 * [upper](./mods/upper.md) aka `up`
 * [lower](./mods/lower.md) aka `low`
 * [date_format](./mods/date_format.md)
@@ -20,15 +22,16 @@ Documentation
 * [escape](./mods/escape.md) aka `e`
 * [unescape](./mods/unescape.md)
 * [strip](./mods/strip.md)
-* [length](./mods/lenght.md)
+* [length](./mods/length.md)
 * [in](./mods/in.md)
 * allowed functions: `json_encode`, `json_decode`, `count`, `is_string`, `is_array`, `is_numeric`, `is_int`, `is_object`,
 `strtotime`, `gettype`, `is_double`, `ip2long`, `long2ip`, `strip_tags`, `nl2br`
-
+* or [add](./ext/mods.md) your own
-[Using](./syntax.md#modifiers) and [addition](./ext/mods.md) of modifiers.
 
 ### Tags
 
+[Usage](./syntax.md#tags)
+
 * [var](./tags/var.md)
 * [if](./tags/if.md), `elseif` and `else`
 * [foreach](./tags/foreach.md), `foreaelse`, `break` and `continue`
@@ -37,12 +40,11 @@ Documentation
 * [cycle](./tags/cycle.md)
 * [include](./tags/include.md)
 * [extends](./tags/extends.md), `use`, `block` and `parent`
-* [capture](./tags/capture.md)
 * [filter](./tags/filter.md)
 * [ignore](./tags/ignore.md)
 * [macro](./tags/macro.md) and `import`
-
+* [autotrim](./tags/autotrim.md)
-[Using](./syntax.md#tags) and [addition](./ext/tags.md) of tags.
+* or [add](./ext/tags.md) your own
 
 ### Extends
 

src/Cytro.php

@@ -91,6 +91,7 @@ class Cytro {
         "e"           => 'Cytro\Modifier::escape', // alias of escape
         "unescape"    => 'Cytro\Modifier::unescape',
         "strip"       => 'Cytro\Modifier::strip',
+        "length"      => 'Cytro\Modifier::length',
         "default"     => 'Cytro\Modifier::defaultValue'
     );
 
@@ -224,6 +225,7 @@ class Cytro {
             throw new InvalidArgumentException("Source must be a valid path or provider object");
         }
         $cytro = new static($provider);
+	    /* @var Cytro $cytro */
         $cytro->setCompileDir($compile_dir);
         if($options) {
             $cytro->setOptions($options);
@@ -525,8 +527,8 @@ class Cytro {
      */
     public function getProvider($scm = false) {
         if($scm) {
-            if(isset($this->_provider[$scm])) {
+            if(isset($this->_providers[$scm])) {
-                return $this->_provider[$scm];
+                return $this->_providers[$scm];
             } else {
                 throw new InvalidArgumentException("Provider for '$scm' not found");
             }
@@ -565,6 +567,23 @@ class Cytro {
         return $this->getTemplate($template)->fetch($vars);
     }
 
+	/**
+	 *
+	 *
+	 * @param string $template name of template
+	 * @param array $vars
+	 * @param $callback
+	 * @param float $chunk
+	 * @return \Cytro\Render
+	 * @example $cytro->pipe("products.yml.tpl", $iterators, [new SplFileObject("/tmp/products.yml"), "fwrite"], 512*1024)
+	 */
+	public function pipe($template, array $vars, $callback, $chunk = 1e6) {
+		ob_start($callback, $chunk, true);
+		$this->getTemplate($template)->display($vars);
+		ob_end_flush();
+
+	}
+
     /**
      * Get template by name
      *

src/Cytro/Compiler.php

@@ -411,6 +411,7 @@ class Compiler {
      */
     public static function extendBody(&$body, $tpl) {
         $t = $tpl;
+//	    var_dump("$tpl: ".$tpl->getBody());
         while(isset($t->_extends)) {
             $t = $t->_extends;
             if(is_object($t)) {
@@ -484,7 +485,7 @@ class Compiler {
             if($scope["name"]) { // is scalar name
                 if($tpl->_compatible) { // is compatible mode
                     $scope->replaceContent(
-                        '<?php /* Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(empty($tpl->b['.$scope["cname"].'])) { '.
+                        '<?php /* 1) Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(empty($tpl->b['.$scope["cname"].'])) { '.
                             '$tpl->b['.$scope["cname"].'] = function($tpl) { ?>'.PHP_EOL.
                                 $scope->getContent().
                             "<?php };".
@@ -493,7 +494,7 @@ class Compiler {
                 } elseif(!isset($tpl->blocks[ $scope["name"] ])) { // is block not registered
                     $tpl->blocks[ $scope["name"] ] = $scope->getContent();
                     $scope->replaceContent(
-                        '<?php /* Block '.$tpl.': '.$scope["cname"].' '.$tpl->_compatible.' */'.PHP_EOL.' $tpl->b['.$scope["cname"].'] = function($tpl) { ?>'.PHP_EOL.
+                        '<?php /* 2) Block '.$tpl.': '.$scope["cname"].' '.$tpl->_compatible.' */'.PHP_EOL.' $tpl->b['.$scope["cname"].'] = function($tpl) { ?>'.PHP_EOL.
                             $scope->getContent().
                         "<?php }; ?>".PHP_EOL
                     );
@@ -501,7 +502,7 @@ class Compiler {
             } else { // dynamic name
                 $tpl->_compatible = true; // enable compatible mode
                 $scope->replaceContent(
-                    '<?php /* Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(empty($tpl->b['.$scope["cname"].'])) { '.
+                    '<?php /* 3) Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(empty($tpl->b['.$scope["cname"].'])) { '.
                         '$tpl->b['.$scope["cname"].'] = function($tpl) { ?>'.PHP_EOL.
                             $scope->getContent().
                         "<?php };".
@@ -512,7 +513,7 @@ class Compiler {
             if(isset($tpl->blocks[ $scope["name"] ])) { // has block
                 if($tpl->_compatible) { // compatible mode enabled
                     $scope->replaceContent(
-                        '<?php /* Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(isset($tpl->b['.$scope["cname"].'])) { echo $tpl->b['.$scope["cname"].']->__invoke($tpl); } else {?>'.PHP_EOL.
+                        '<?php /* 4) Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(isset($tpl->b['.$scope["cname"].'])) { echo $tpl->b['.$scope["cname"].']->__invoke($tpl); } else {?>'.PHP_EOL.
                             $tpl->blocks[ $scope["name"] ].
                         '<?php } ?>'.PHP_EOL
                     );
@@ -520,9 +521,11 @@ class Compiler {
                 } else {
                     $scope->replaceContent($tpl->blocks[ $scope["name"] ]);
                 }
+//            } elseif(isset($tpl->_extended) || !empty($tpl->_compatible)) {
             } elseif(isset($tpl->_extended) && $tpl->_compatible || empty($tpl->_extended)) {
+//	            var_dump("$tpl: exxx");
                 $scope->replaceContent(
-                    '<?php /* Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(isset($tpl->b['.$scope["cname"].'])) { echo $tpl->b['.$scope["cname"].']->__invoke($tpl); } else {?>'.PHP_EOL.
+                    '<?php /* 5) Block '.$tpl.': '.$scope["cname"].' */'.PHP_EOL.' if(isset($tpl->b['.$scope["cname"].'])) { echo $tpl->b['.$scope["cname"].']->__invoke($tpl); } else {?>'.PHP_EOL.
                         $scope->getContent().
                     '<?php } ?>'.PHP_EOL
                 );
@@ -629,74 +632,59 @@ class Compiler {
         return 'array('.implode(",", $_code).')';
     }
 
-    public static function varOpen(Tokenizer $tokens, Scope $scope) {
+	/**
-        $scope->is_closed = true;
+	 * @param Tokenizer $tokens
-        return self::setVar($tokens, $scope->tpl).';';
+	 * @param Scope $scope
+	 * @return string
+	 */
+	public static function varOpen(Tokenizer $tokens, Scope $scope) {
+	    $var = $scope->tpl->parseVariable($tokens, Template::DENY_MODS);
+	    if($tokens->is('=')) { // inline tag {var ...}
+		    $scope->is_closed = true;
+		    $tokens->next();
+		    if($tokens->is("[")) {
+			    return $var.'='.$scope->tpl->parseArray($tokens);
+		    } else {
+			    return $var.'='.$scope->tpl->parseExp($tokens, true);
+		    }
+	    } else {
+		    $scope["name"] = $var;
+		    if($tokens->is('|')) {
+		        $scope["value"] = $scope->tpl->parseModifier($tokens, "ob_get_clean()");
+		    } else {
+			    $scope["value"] = "ob_get_clean()";
+		    }
+		    return 'ob_start();';
+	    }
     }
 
-    public static function varClose() {
+	/**
-        return '';
+	 * @param Tokenizer $tokens
+	 * @param Scope $scope
+	 * @return string
+	 */
+	public static function varClose(Tokenizer $tokens, Scope $scope) {
+        return $scope["name"].'='.$scope["value"].';';
     }
 
-    /**
-     * Tag {var ...}
-     *
-     * @static
-     * @param Tokenizer $tokens
-     * @param Template           $tpl
-     * @return string
-     */
-    //public static function assign(Tokenizer $tokens, Template $tpl) {
-      //  return self::setVar($tokens, $tpl).';';
-    //}
-
-    /**
-     * Set variable expression
-     * @param Tokenizer $tokens
-     * @param Template $tpl
-     * @param bool $allow_array
-     * @return string
-     */
-    public static function setVar(Tokenizer $tokens, Template $tpl, $allow_array = true) {
-        $var = $tpl->parseVariable($tokens, $tpl::DENY_MODS);
-
-        $tokens->get('=');
-        $tokens->next();
-        if($tokens->is("[") && $allow_array) {
-            return $var.'='.$tpl->parseArray($tokens);
-        } else {
-            return $var.'='.$tpl->parseExp($tokens, true);
-        }
-    }
-
-    public static function filterOpen(Tokenizer $tokens, Scope $scope) {
-        $scope["filter"] = $scope->tpl->parseModifier($tokens, "ob_get_clean()");
-        return "ob_start();";
-    }
-
-    public static function filterClose($tokens, Scope $scope) {
-        return "echo ".$scope["filter"].";";
-    }
 
     /**
      * @param Tokenizer $tokens
      * @param Scope $scope
      * @return string
      */
-    public static function captureOpen(Tokenizer $tokens, Scope $scope) {
+    public static function filterOpen(Tokenizer $tokens, Scope $scope) {
-        if($tokens->is("|")) {
+        $scope["filter"] = $scope->tpl->parseModifier($tokens, "ob_get_clean()");
-            $scope["value"] = $scope->tpl->parseModifier($tokens, "ob_get_clean()");
-        } else {
-            $scope["value"] = "ob_get_clean()";
-        }
-
-        $scope["var"] = $scope->tpl->parseVariable($tokens, Template::DENY_MODS);
-
         return "ob_start();";
     }
 
-    public static function captureClose($tokens, Scope $scope) {
+    /**
-        return $scope["var"]." = ".$scope["value"].";";
+     * @param $tokens
+     * @param Scope $scope
+     * @return string
+     */
+    public static function filterClose($tokens, Scope $scope) {
+        return "echo ".$scope["filter"].";";
     }
 
     /**

src/Cytro/Modifier.php

@@ -80,7 +80,7 @@ class Modifier {
     }
 
     /**
-     * Crop string by length (support unicode)
+     * Crop string to specific length (support unicode)
      *
      * @param string $string text witch will be truncate
      * @param int $length maximum symbols of result string
@@ -134,7 +134,7 @@ class Modifier {
      * @return int
      */
     public static function length($item) {
-        if(is_scalar($item)) {
+        if(is_string($item)) {
             return strlen(preg_replace('#[\x00-\x7F]|[\x80-\xDF][\x00-\xBF]|[\xE0-\xEF][\x00-\xBF]{2}#s', ' ', $item));
         } elseif (is_array($item)) {
             return count($item);

src/Cytro/Template.php

@@ -97,12 +97,12 @@ class Template extends Render {
         $this->_name = $name;
         if($provider = strstr($name, ":", true)) {
             $this->_scm = $provider;
-            $this->_base_name = substr($name, strlen($provider));
+            $this->_base_name = substr($name, strlen($provider) + 1);
         } else {
-            $this->_base_name = $name;
+	        $this->_base_name = $name;
         }
-        $this->_provider = $this->_cytro->getProvider($provider);
+	    $this->_provider = $this->_cytro->getProvider($provider);
-        $this->_src = $this->_provider->getSource($name, $this->_time);
+        $this->_src = $this->_provider->getSource($this->_base_name, $this->_time);
         if($compile) {
             $this->compile();
         }
@@ -134,56 +134,61 @@ class Template extends Render {
         if(!isset($this->_src)) { // already compiled
             return;
         }
-        $pos = 0;
+	    $end = $pos = 0;
-        $frag = "";
         while(($start = strpos($this->_src, '{', $pos)) !== false) { // search open-symbol of tags
-            switch($this->_src[$start + 1]) { // check next char
+            switch($this->_src[$start + 1]) { // check next character
                 case "\n": case "\r": case "\t": case " ": case "}": // ignore the tag
                     $pos = $start + 1; // try find tags after the current char
                     continue 2;
                 case "*": // if comment block
                     $end = strpos($this->_src, '*}', $start); // finding end of the comment block
+					if($end === false) {
+						throw new CompileException("Unclosed comment block in line {$this->_line}", 0, 1, $this->_name, $this->_line);
+					}
                     $_frag = substr($this->_src, $this->_pos, $start - $end); // read the comment block for precessing
-                    $this->_line += substr_count($_frag, "\n"); // count skipped lines
+                    $this->_line += substr_count($_frag, "\n"); // count skipped lines in comment block
-                    $pos = $end + 1; // trying finding tags after the comment block
+                    $pos = $end + 1; // seek pointer
                     continue 2;
             }
-            $end = strpos($this->_src, '}', $start); // search close-symbol of the tag
+	        $frag = substr($this->_src, $pos, $start - $pos);  // variable $frag contains chars after previous '}' and current '{'
-            if(!$end) { // if unexpected end of template
+	        $this->_appendText($frag);
-                throw new CompileException("Unclosed tag in line {$this->_line}", 0, 1, $this->_name, $this->_line);
-            }
-            $frag .= substr($this->_src, $this->_pos, $start - $this->_pos);  // variable $frag contains chars after previous '}' and current '{'
-            $tag = substr($this->_src, $start, $end - $start + 1); // variable $tag contains cytro tag '{...}'
-            $this->_line += substr_count($this->_src, "\n", $this->_pos, $end - $start + 1); // count lines in $frag and $tag (using original text $code)
-            $pos = $this->_pos = $end + 1; // move search-pointer to end of the tag
 
-            if($tag[strlen($tag) - 2] === "-") { // check right trim flag
+	        $from = $start;
-                $_tag = substr($tag, 1, -2);
+	        reparse: { // yep, i use goto operator. For this algorithm it is good choice
-                $_frag = rtrim($frag);
+		        $end = strpos($this->_src, '}', $from); // search close-symbol of the tag
-            } else {
+		        if($end === false) { // if unexpected end of template
-                $_tag = substr($tag, 1, -1);
+	                throw new CompileException("Unclosed tag in line {$this->_line}", 0, 1, $this->_name, $this->_line);
-                $_frag = $frag;
-            }
-            if($this->_ignore) { // check ignore
-                if($_tag === '/ignore') {
-                    $this->_ignore = false;
-                    $this->_appendText($_frag);
-                } else { // still ignore
-                    $frag .= $tag;
-                    continue;
-                }
-            } else {
-                $this->_appendText($_frag);
-                $tokens = new Tokenizer($_tag);
-                $this->_appendCode($this->_tag($tokens), $tag);
-                if($tokens->key()) { // if tokenizer have tokens - throws exceptions
-                    throw new CompileException("Unexpected token '".$tokens->current()."' in {$this} line {$this->_line}, near '{".$tokens->getSnippetAsString(0,0)."' <- there", 0, E_ERROR, $this->_name, $this->_line);
                 }
+                $tag = substr($this->_src, $start, $end - $start + 1); // variable $tag contains cytro tag '{...}'
 
-            }
+		        $_tag = substr($tag, 1, -1); // strip delimiters '{' and '}'
-            $frag = "";
+
+		        if($this->_ignore) { // check ignore
+			        if($_tag === '/ignore') { // turn off ignore
+				        $this->_ignore = false;
+			        } else { // still ignore
+				        $this->_appendText($tag);
+			        }
+			        $pos = $start + strlen($tag);
+			        continue;
+		        } else {
+			        $tokens = new Tokenizer($_tag); // tokenize the tag
+			        if($tokens->isIncomplete()) { // all strings finished?
+				        $from = $end + 1;
+				        goto reparse; // find another close-symbol
+			        }
+			        $this->_appendCode( $this->_tag($tokens) , $tag); // start the tag lexer
+			        $pos = $end + 1; // move search-pointer to end of the tag
+			        if($tokens->key()) { // if tokenizer have tokens - throws exceptions
+				        throw new CompileException("Unexpected token '".$tokens->current()."' in {$this} line {$this->_line}, near '{".$tokens->getSnippetAsString(0,0)."' <- there", 0, E_ERROR, $this->_name, $this->_line);
+			        }
+
+			    }
+		    }
+            unset($frag);
         }
-        $this->_appendText(substr($this->_src, $this->_pos));
+        gc_collect_cycles();
+        $this->_appendText(substr($this->_src, $end ? $end + 1 : 0));
         if($this->_stack) {
             $_names = array();
             $_line = 0;
@@ -191,9 +196,9 @@ class Template extends Render {
                 if(!$_line) {
                     $_line = $scope->line;
                 }
-                $_names[] = '{'.$scope->name.'} defined on line '.$scope->line;
+                $_names[] = '{'.$scope->name.'} opened on line '.$scope->line;
             }
-            throw new CompileException("Unclosed tag(s): ".implode(", ", $_names), 0, 1, $this->_name, $_line);
+            throw new CompileException("Unclosed tag".(count($_names) == 1 ? "" : "s").": ".implode(", ", $_names), 0, 1, $this->_name, $_line);
         }
         unset($this->_src);
         if($this->_post) {
@@ -217,6 +222,7 @@ class Template extends Render {
      * @param string $text
      */
     private function _appendText($text) {
+	    $this->_line += substr_count($text, "\n");
         if($this->_filter) {
             if(strpos($text, "<?") === false) {
                 $this->_body .= $text;
@@ -262,9 +268,11 @@ class Template extends Render {
      * @param $source
      */
     private function _appendCode($code, $source) {
+
         if(!$code) {
             return;
         } else {
+	        $this->_line += substr_count($source, "\n");
             if(strpos($code, '?>') !== false) {
                 $code = $this->_escapeCode($code); // paste PHP_EOL
             }
@@ -357,14 +365,14 @@ class Template extends Render {
         return parent::fetch($values);
     }
 
-    /**
+	/**
-     * Internal tags router
+	 * Internal tags router
-     * @param Tokenizer $tokens
+	 * @param Tokenizer $tokens
-     * @throws UnexpectedTokenException
+	 *
-     * @throws CompileException
+	 * @throws SecurityException
-     * @throws SecurityException
+	 * @throws CompileException
-     * @return string executable PHP code
+	 * @return string executable PHP code
-     */
+	 */
     private function _tag(Tokenizer $tokens) {
         try {
             if($tokens->is(Tokenizer::MACRO_STRING)) {
@@ -742,104 +750,68 @@ class Template extends Render {
         return $_scalar;
     }
 
-    /**
+	/**
-     * Parse string with or without variable
+	 * Parse string with or without variable
-     *
+	 *
-     * @param Tokenizer $tokens
+	 * @param Tokenizer $tokens
-     * @throws UnexpectedTokenException
+	 * @throws UnexpectedTokenException
-     * @return string
+	 * @return string
-     */
+	 */
     public function parseSubstr(Tokenizer $tokens) {
-        ref: {
+        if($tokens->is('"',"`")) {
-            if($tokens->is('"',"`")) {
+            $stop = $tokens->current();
-                $p = $tokens->p;
+            $_str = '"';
-                $stop = $tokens->current();
+            $tokens->next();
-                $_str = '"';
+            while($t = $tokens->key()) {
-                $tokens->next();
+                if($t === T_ENCAPSED_AND_WHITESPACE) {
-                while($t = $tokens->key()) {
+                    $_str .= $tokens->current();
-                    if($t === T_ENCAPSED_AND_WHITESPACE) {
+                    $tokens->next();
-                        $_str .= $tokens->current();
+                } elseif($t === T_VARIABLE) {
-                        $tokens->next();
+                    if(strlen($_str) > 1) {
-                    } elseif($t === T_VARIABLE) {
+                        $_str .= '".';
-                        if(strlen($_str) > 1) {
-                            $_str .= '".';
-                        } else {
-                            $_str = "";
-                        }
-                        $_str .= '$tpl["'.substr($tokens->current(), 1).'"]';
-                        $tokens->next();
-                        if($tokens->is($stop)) {
-                            $tokens->skip();
-                            return $_str;
-                        } else {
-                            $_str .= '."';
-                        }
-                    } elseif($t === T_CURLY_OPEN) {
-                        if(strlen($_str) > 1) {
-                            $_str .= '".';
-                        } else {
-                            $_str = "";
-                        }
-                        $tokens->getNext(T_VARIABLE);
-                        $_str .= '('.$this->parseExp($tokens).')';
-                        /*if(!$tokens->valid()) {
-                            $more = $this->_getMoreSubstr($stop);
-                            //var_dump($more); exit;
-                            $tokens->append("}".$more, $p);
-                            var_dump("Curly", $more, $tokens->getSnippetAsString());
-                            exit;
-                        }*/
-
-                        //$tokens->skip('}');
-                        if($tokens->is($stop)) {
-                            $tokens->next();
-                            return $_str;
-                        } else {
-                            $_str .= '."';
-                        }
-                    } elseif($t === "}") {
-                        $tokens->next();
-                    } elseif($t === $stop) {
-                        $tokens->next();
-                        return $_str.'"';
                     } else {
-
+                        $_str = "";
-                        break;
                     }
-                }
+                    $_str .= '$tpl["'.substr($tokens->current(), 1).'"]';
-                if($more = $this->_getMoreSubstr($stop)) {
+                    $tokens->next();
-                    $tokens->append("}".$more, $p);
+                    if($tokens->is($stop)) {
-                    goto ref;
+                        $tokens->skip();
-                }
+                        return $_str;
-                throw new UnexpectedTokenException($tokens);
+                    } else {
-            } elseif($tokens->is(T_CONSTANT_ENCAPSED_STRING)) {
+                        $_str .= '."';
-                return $tokens->getAndNext();
+                    }
-            } elseif($tokens->is(T_ENCAPSED_AND_WHITESPACE)) {
+                } elseif($t === T_CURLY_OPEN) {
-                $p = $tokens->p;
+                    if(strlen($_str) > 1) {
-                if($more = $this->_getMoreSubstr($tokens->curr[1][0])) {
+                        $_str .= '".';
-                    $tokens->append("}".$more, $p);
+                    } else {
-                    goto ref;
+                        $_str = "";
-                }
+                    }
-                throw new UnexpectedTokenException($tokens);
+                    $tokens->getNext(T_VARIABLE);
-            } else {
+                    $_str .= '('.$this->parseExp($tokens).')';
-                return "";
+                    if($tokens->is($stop)) {
-            }
+                        $tokens->next();
-        }
+                        return $_str;
-    }
+                    } else {
+                        $_str .= '."';
+                    }
+                } elseif($t === "}") {
+                    $tokens->next();
+                } elseif($t === $stop) {
+                    $tokens->next();
+                    return $_str.'"';
+                } else {
 
-    /**
+                    break;
-     * @param string $after
+                }
-     * @return bool|string
+            }
-     */
+            throw new UnexpectedTokenException($tokens);
-    private function _getMoreSubstr($after) {
+        } elseif($tokens->is(T_CONSTANT_ENCAPSED_STRING)) {
-        $end = strpos($this->_src, $after, $this->_pos);
+            return $tokens->getAndNext();
-        $end = strpos($this->_src, "}", $end);
+        } elseif($tokens->is(T_ENCAPSED_AND_WHITESPACE)) {
-        if(!$end) {
+            throw new UnexpectedTokenException($tokens);
-            return false;
+        } else {
+            return "";
         }
-        $fragment = substr($this->_src, $this->_pos, $end - $this->_pos);
-        $this->_pos = $end + 1;
-        return $fragment;
     }
 
     /**
@@ -1091,4 +1063,5 @@ class Template extends Render {
 
 class CompileException extends \ErrorException {}
 class SecurityException extends CompileException {}
-class ImproperUseException extends \LogicException {}
+class ImproperUseException extends \LogicException {}
+class ReparseTagException extends \Exception {}

src/Cytro/Tokenizer.php

@@ -79,6 +79,7 @@ class Tokenizer {
 
     public $tokens;
     public $p = 0;
+	public $quotes = 0;
     private $_max = 0;
     private $_last_no = 0;
 
@@ -100,7 +101,7 @@ class Tokenizer {
             \T_NEW => 1,         \T_PRINT => 1,       \T_PRIVATE => 1, \T_PUBLIC => 1,      \T_PROTECTED => 1,   \T_REQUIRE => 1,
             \T_REQUIRE_ONCE => 1,\T_RETURN => 1,      \T_RETURN => 1,  \T_STRING => 1,      \T_SWITCH => 1,      \T_THROW => 1,
             \T_TRAIT => 1,       \T_TRAIT_C => 1,     \T_TRY => 1,     \T_UNSET => 1,       \T_UNSET => 1,       \T_VAR => 1,
-            \T_WHILE => 1
+            \T_WHILE => 1,       \T_YIELD => 1
         ),
         self::MACRO_INCDEC => array(
             \T_INC => 1, \T_DEC => 1
@@ -142,52 +143,56 @@ class Tokenizer {
         'true' => 1, 'false' => 1, 'null' => 1, 'TRUE' => 1, 'FALSE' => 1, 'NULL' => 1
     );
 
-    /**
+	/**
-     * Translate expression to tokens list.
+	 * @param $query
-     *
+	 */
-     * @static
+	public function __construct($query) {
-     * @param string $query
+	    $tokens = array(-1 => array(\T_WHITESPACE, '', '', 1));
-     * @return array
+	    $_tokens = token_get_all("<?php ".$query);
-     */
+	    $line = 1;
-    public static function decode($query) {
+	    array_shift($_tokens);
-        $tokens = array(-1 => array(\T_WHITESPACE, '', '', 1));
+	    $i = 0;
-        $_tokens = token_get_all("<?php ".$query);
+	    foreach($_tokens as $token) {
-        $line = 1;
+		    if(is_string($token)) {
-        array_shift($_tokens);
+			    if($token === '"' || $token === "'" || $token === "`") {
-        $i = 0;
+				    $this->quotes++;
-        foreach($_tokens as &$token) {
+			    }
-            if(is_string($token)) {
+			    $tokens[] = array(
-                $tokens[] = array(
+				    $token,
-                    $token,
+				    $token,
-                    $token,
+				    "",
-                    "",
+				    $line,
-                    $line,
+			    );
-                );
+			    $i++;
-                $i++;
+		    } elseif ($token[0] === \T_WHITESPACE) {
-            } elseif ($token[0] === \T_WHITESPACE) {
+			    $tokens[$i-1][2] = $token[1];
-                $tokens[$i-1][2] = $token[1];
+		    } else {
-            } else {
+			    $tokens[] = array(
-                $tokens[] = array(
+				    $token[0],
-                    $token[0],
+				    $token[1],
-                    $token[1],
+				    "",
-                    "",
+				    $line =  $token[2],
-                    $line =  $token[2],
+				    token_name($token[0]) // debug
-                );
+			    );
-                $i++;
+			    $i++;
-            }
+		    }
 
-        }
+	    }
-
+	    unset($tokens[-1]);
-        return $tokens;
+	    $this->tokens = $tokens;
-    }
-
-    public function __construct($query, $decode = 0) {
-        $this->tokens = self::decode($query, $decode);
-        unset($this->tokens[-1]);
         $this->_max = count($this->tokens) - 1;
         $this->_last_no = $this->tokens[$this->_max][3];
     }
 
+	/**
+	 * Is incomplete mean some string not closed
+	 *
+	 * @return int
+	 */
+	public function isIncomplete() {
+		return ($this->quotes % 2) || ($this->tokens[$this->_max][0] === T_ENCAPSED_AND_WHITESPACE);
+	}
+
     /**
      * Return the current element
      *
@@ -468,7 +473,7 @@ class Tokenizer {
     }
 
     /**
-     * Get tokens near current token
+     * Get tokens near current position
      * @param int $before count tokens before current token
      * @param int $after count tokens after current token
      * @return array
@@ -553,30 +558,6 @@ class Tokenizer {
     public function getLine() {
         return $this->curr ? $this->curr[3] : $this->_last_no;
     }
-
-    /**
-     * Parse code and append tokens. This method move pointer to offset.
-     *
-     * @param string $code
-     * @param int $offset if not -1 replace tokens from position $offset
-     * @return Tokenizer
-     */
-    public function append($code, $offset = -1) {
-        if($offset != -1) {
-            $code = $this->getSubstr($offset).$code;
-            if($this->p > $offset) {
-                $this->p = $offset;
-            }
-
-            $this->tokens = array_slice($this->tokens, 0, $offset);
-        }
-        $tokens = self::decode($code);
-        unset($tokens[-1], $this->prev, $this->curr, $this->next);
-        $this->tokens = array_merge($this->tokens, $tokens);
-        $this->_max = count($this->tokens) - 1;
-        $this->_last_no = $this->tokens[$this->_max][3];
-        return $this;
-    }
 }
 
 /**

tests/TestCase.php

@@ -24,6 +24,11 @@ class TestCase extends \PHPUnit_Framework_TestCase {
             FS::clean(CYTRO_RESOURCES.'/compile/');
         }
         $this->cytro = Cytro::factory(CYTRO_RESOURCES.'/template', CYTRO_RESOURCES.'/compile');
+        $this->cytro->addModifier('dots', __CLASS__.'::dots');
+    }
+
+    public static function dots($value) {
+        return $value."...";
     }
 
     public static function setUpBeforeClass() {
@@ -85,10 +90,105 @@ class TestCase extends \PHPUnit_Framework_TestCase {
         $this->fail("Code $code must be invalid");
     }
 
-    public function assertRender($tpl, $result) {
+    public function assertRender($tpl, $result, $debug = false) {
         $template = $this->cytro->compileCode($tpl);
+        if($debug) {
+            print_r("$tpl:\n".$template->getBody());
+        }
         $this->assertSame($result, $template->fetch($this->values));
     }
+
+
+	public static function providerNumbers() {
+		return array(
+			array('77', 77),
+			array('-33', -33),
+			array('0.2', 0.2),
+			array('-0.3', -0.3),
+			array('1e6', 1e6),
+			array('-2e6', -2e6),
+		);
+	}
+
+    public static function providerStrings() {
+        return array(
+            array('"str"', 'str'),
+            array('"str\nand\nmany\nlines"', "str\nand\nmany\nlines"),
+            array('"str and \'substr\'"', "str and 'substr'"),
+            array('"str and \"substr\""', 'str and "substr"'),
+            array("'str'", 'str'),
+            array("'str\\nin\\none\\nline'", 'str\nin\none\nline'),
+            array("'str and \"substr\"'", 'str and "substr"'),
+            array("'str and \'substr\''", "str and 'substr'"),
+            array('"$one"', '1'),
+            array('"$one $two"', '1 2'),
+            array('"$one and $two"', '1 and 2'),
+            array('"a $one and $two b"', 'a 1 and 2 b'),
+            array('"{$one}"', '1'),
+            array('"a {$one} b"', 'a 1 b'),
+            array('"{$one + 2}"', '3'),
+            array('"{$one * $two + 1}"', '3'),
+            array('"{$one} and {$two}"', '1 and 2'),
+            array('"$one and {$two}"', '1 and 2'),
+            array('"{$one} and $two"', '1 and 2'),
+            array('"a {$one} and {$two} b"', 'a 1 and 2 b'),
+            array('"{$one+1} and {$two-1}"', '2 and 1'),
+            array('"a {$one+1} and {$two-1} b"', 'a 2 and 1 b'),
+            array('"a {$one|dots} and {$two|dots} b"', 'a 1... and 2... b'),
+            array('"a {$one|dots} and $two b"', 'a 1... and 2 b'),
+            array('"a $one and {$two|dots} b"', 'a 1 and 2... b'),
+        );
+    }
+
+	public function providerVariables() {
+		return array();
+	}
+
+	public static function providerObjects() {
+		return array();
+	}
+
+	public static function providerArrays() {
+		$scalars = array();
+		$data = array(
+			array('[]', array()),
+			array('[[],[]]', array(array(), array())),
+		);
+		foreach(self::providerScalars() as $scalar) {
+			$scalars[0][] = $scalar[0];
+			$scalars[1][] = $scalar[1];
+
+			$data[] = array(
+				"[".$scalar[0]."]",
+				array($scalar[1])
+			);
+			$data[] = array(
+				"['some_key' =>".$scalar[0]."]",
+				array('some_key' => $scalar[1])
+			);
+		}
+		$data[] = array(
+			"[".implode(", ", $scalars[0])."]",
+			$scalars[1]
+		);
+		return $data;
+	}
+
+	public static function providerScalars() {
+		return array_merge(
+			self::providerNumbers(),
+			self::providerStrings()
+		);
+	}
+
+	public static function providerValues() {
+		return array_merge(
+			self::providerScalars(),
+			self::providerArrays(),
+			self::providerVariables(),
+			self::providerObjects()
+		);
+	}
 }
 
 class Fake implements \ArrayAccess {
@@ -113,4 +213,8 @@ class Fake implements \ArrayAccess {
     public function offsetUnset($offset) {
         unset($this->vars[$offset]);
     }
+
+	public function proxy() {
+		return implode(", ", func_get_args());
+	}
 }

tests/cases/Cytro/CustomProvider.php

@@ -1,17 +0,0 @@
-<?php
-
-namespace Cytro;
-
-
-class CustomProvider extends TestCase {
-
-    public function setUp() {
-        $this->setUp();
-        $this->cytro->addProvider("my", new FSProvider(CYTRO_RESOURCES.'/provider'));
-    }
-
-    public function testCustom() {
-        $this->render("start: {include 'my:include.tpl'}", 'start: include template');
-        $this->render("start: {import 'my:macros.tpl' as ops} {ops.add a=3 b=6}");
-    }
-}

tests/cases/Cytro/CustomProviderTest.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Cytro;
+
+
+class CustomProviderTest extends TestCase {
+
+    public function setUp() {
+        parent::setUp();
+        $this->cytro->addProvider("my", new FSProvider(CYTRO_RESOURCES.'/provider'));
+    }
+
+    public function testCustom() {
+        $this->assertRender("start: {include 'my:include.tpl'}", 'start: include template');
+        //$this->assertRender("start: {import 'my:macros.tpl' as ops} {ops.add a=3 b=6}");
+    }
+}

tests/cases/Cytro/ModifiersTest.php

@@ -5,8 +5,8 @@ namespace Cytro;
 class ModifiersTest extends TestCase {
 
     public static function providerTruncate() {
-        $lorem = 'Lorem ipsum dolor sit amet';
+        $lorem = 'Lorem ipsum dolor sit amet'; // en
-        $uni   = 'Лорем ипсум долор сит амет';
+        $uni   = 'Лорем ипсум долор сит амет'; // ru
         return array(
             // ascii chars
             array($lorem, 'Lorem ip...', 8),
@@ -23,13 +23,15 @@ class ModifiersTest extends TestCase {
         );
     }
 
+
     /**
      * @dataProvider providerTruncate
      * @param $in
      * @param $out
      * @param $count
      * @param string $delim
-     * @param bool $by_word
+     * @param bool $by_words
+     * @param bool $middle
      */
     public function testTruncate($in, $out, $count, $delim = '...', $by_words = false, $middle = false) {
         $tpl = $this->cytro->compileCode('{$text|truncate:$count:$delim:$by_words:$middle}');
@@ -41,4 +43,58 @@ class ModifiersTest extends TestCase {
             "middle" => $middle
         )));
     }
+
+    public static function providerUpLow() {
+        return array(
+            array("up", "lorem", "LOREM"),
+            array("up", "Lorem", "LOREM"),
+            array("up", "loREM", "LOREM"),
+            array("up", "223a", "223A"),
+            array("low", "lorem", "lorem"),
+            array("low", "Lorem", "lorem"),
+            array("low", "loREM", "lorem"),
+            array("low", "223A", "223a"),
+        );
+    }
+
+
+    /**
+     * @dataProvider providerUpLow
+     * @param $modifier
+     * @param $in
+     * @param $out
+     */
+    public function testUpLow($modifier, $in, $out) {
+        $tpl = $this->cytro->compileCode('{$text|'.$modifier.'}');
+        $this->assertEquals($out, $tpl->fetch(array(
+            "text" => $in,
+        )));
+    }
+
+    public static function providerLength() {
+        return array(
+            array("length", 6),
+            array("длина", 5),
+            array("length - длина", 14),
+            array(array(1, 33, "c" => 4), 3),
+            array(new \ArrayIterator(array(1, "c" => 4)), 2),
+            array(true, 0),
+            array(new \stdClass(), 0),
+            array(5, 0)
+        );
+    }
+
+    /**
+     * @dataProvider providerLength
+     * @param $in
+     * @param $in
+     * @param $out
+     */
+    public function testLength($in, $out) {
+        $tpl = $this->cytro->compileCode('{$data|length}');
+        $this->assertEquals($out, $tpl->fetch(array(
+            "data" => $in,
+        )));
+    }
+
 }

tests/cases/Cytro/TagsTest.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace Cytro;
+
+
+class TagsTest extends TestCase {
+
+//	public function _testSandbox() {
+//		try {
+//			var_dump($this->cytro->compileCode(" literal: { \$a} end")->getBody());
+//		} catch(\Exception $e) {
+//			echo "$e";
+//		}
+//		exit;
+//	}
+
+    /**
+     * @dataProvider providerScalars
+     */
+    public function testVar($tpl_val, $val) {
+        $this->assertRender("{var \$a=$tpl_val}\nVar: {\$a}", "\nVar: ".$val);
+    }
+
+    /**
+     * @dataProvider providerScalars
+     */
+    public function testVarBlock($tpl_val, $val) {
+        $this->assertRender("{var \$a}before {{$tpl_val}} after{/var}\nVar: {\$a}", "\nVar: before ".$val." after");
+    }
+
+    /**
+     * @dataProvider providerScalars
+     */
+    public function testVarBlockModified($tpl_val, $val) {
+        $this->assertRender("{var \$a|low|dots}before {{$tpl_val}} after{/var}\nVar: {\$a}", "\nVar: ".strtolower("before ".$val." after")."...");
+    }
+
+    public function testCycle() {
+
+    }
+
+
+    public function testFilter() {
+
+    }
+
+}

tests/cases/Cytro/TemplateTest.php

@@ -4,6 +4,11 @@ use Cytro\Template,
     Cytro,
     Cytro\Render;
 
+/**
+ * Test template parsing
+ *
+ * @package Cytro
+ */
 class TemplateTest extends TestCase {
 
     public function setUp() {
@@ -15,11 +20,6 @@ class TemplateTest extends TestCase {
         )));
     }
 
-    /*public function testSandbox() {
-        var_dump($this->cytro->compileCode('{"$s:{$b+1}f d {$d}"}')->_body);
-        exit;
-    }*/
-
     public static function providerVars() {
         $a = array("a" => "World");
         $obj = new \stdClass;
@@ -73,41 +73,6 @@ class TemplateTest extends TestCase {
         );
     }
 
-    public static function providerScalars() {
-        return array(
-            array('77', 77),
-            array('-33', -33),
-            array('0.2', 0.2),
-            array('-0.3', -0.3),
-            array('1e6', 1e6),
-            array('-2e6', -2e6),
-            array('"str"', 'str'),
-            array('"str\nand\nmany\nlines"', "str\nand\nmany\nlines"),
-            array('"str and \'substr\'"', "str and 'substr'"),
-            array('"str and \"substr\""', 'str and "substr"'),
-            array("'str'", 'str'),
-            array("'str\\nin\\none\\nline'", 'str\nin\none\nline'),
-            array("'str and \"substr\"'", 'str and "substr"'),
-            array("'str and \'substr\''", "str and 'substr'"),
-            array('"$one"', '1'),
-            array('"$one $two"', '1 2'),
-            array('"$one and $two"', '1 and 2'),
-            array('"a $one and $two b"', 'a 1 and 2 b'),
-            array('"{$one}"', '1'),
-            array('"a {$one} b"', 'a 1 b'),
-            array('"{$one + 2}"', '3'),
-            array('"{$one * $two + 1}"', '3'),
-            array('"{$one} and {$two}"', '1 and 2'),
-            array('"$one and {$two}"', '1 and 2'),
-            array('"{$one} and $two"', '1 and 2'),
-            array('"a {$one} and {$two} b"', 'a 1 and 2 b'),
-            array('"{$one+1} and {$two-1}"', '2 and 1'),
-            array('"a {$one+1} and {$two-1} b"', 'a 2 and 1 b'),
-            array('"a {$one|dots} and {$two|dots} b"', 'a 1... and 2... b'),
-            array('"a {$one|dots} and $two b"', 'a 1... and 2 b'),
-            array('"a $one and {$two|dots} b"', 'a 1 and 2... b'),
-        );
-    }
 
     public static function providerVarsInvalid() {
         return array(
@@ -347,7 +312,7 @@ class TemplateTest extends TestCase {
 
     public static function providerCreateVarInvalid() {
         return array(
-            array('Create: {var $v} Result: {$v} end',               'Cytro\CompileException', "Unexpected end of expression"),
+            array('Create: {var $v} Result: {$v} end',               'Cytro\CompileException', "Unclosed tag: {var} opened"),
             array('Create: {var $v = } Result: {$v} end',            'Cytro\CompileException', "Unexpected end of expression"),
             array('Create: {var $v = 1++} Result: {$v} end',         'Cytro\CompileException', "Unexpected token '++'"),
             array('Create: {var $v = c} Result: {$v} end',           'Cytro\CompileException', "Unexpected token 'c'"),
@@ -586,7 +551,7 @@ class TemplateTest extends TestCase {
             array('Layers: {for $a=4 to=6} block1 {if 1}  {/for} {/if} end',                        'Cytro\CompileException', "Unexpected closing of the tag 'for'"),
             array('Layers: {switch 1} {if 1} {case 1} {/if} {/switch} end',                         'Cytro\CompileException', "Unexpected tag 'case' (this tag can be used with 'switch')"),
             array('Layers: {/switch} end',                                                          'Cytro\CompileException', "Unexpected closing of the tag 'switch'"),
-            array('Layers: {if 1} end',                                                             'Cytro\CompileException', "Unclosed tag(s): {if}"),
+            array('Layers: {if 1} end',                                                             'Cytro\CompileException', "Unclosed tag: {if}"),
         );
     }