comments WIP

src/main/java/carpet/script/Tokenizer.java

@@ -22,6 +22,8 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
     private int pos = 0;
     private int lineno = 0;
     private int linepos = 0;
+    private boolean comments;
+    private boolean newLinesMarkers;
 
 
     /** The original input expression. */
@@ -31,10 +33,12 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
 
     private Expression expression;
 
-    Tokenizer(Expression expr, String input)
+    Tokenizer(Expression expr, String input, boolean allowComments, boolean allowNewLineMakers)
     {
         this.input = input;
         this.expression = expr;
+        this.comments = allowComments;
+        this.newLinesMarkers = allowNewLineMakers;
     }
 
 
@@ -214,6 +218,22 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
                     && (pos < input.length()))
             {
                 greedyMatch += ch;
+                if (comments && "//".equals(greedyMatch))
+                {
+
+                    while ( ch != '\n' && pos < input.length())
+                    {
+                        ch = input.charAt(pos++);
+                        linepos++;
+                        greedyMatch += ch;
+
+                    }
+                    if (ch=='\n')
+                    {
+                        lineno++;
+                        linepos = 0;
+                    }
+                }
                 pos++;
                 linepos++;
                 if (Expression.none.isAnOperator(greedyMatch))
@@ -222,6 +242,14 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
                 }
                 ch = pos == input.length() ? 0 : input.charAt(pos);
             }
+            if (newLinesMarkers && "$".equals(greedyMatch))
+            {
+                lineno++;
+                linepos = 0;
+                token.type = Token.TokenType.MARKER;
+                token.append('$');
+                return token; // skipping previous token lookback
+            }
             if (validOperatorSeenUntil != -1)
             {
                 token.append(input.substring(initialPos, validOperatorSeenUntil));
@@ -277,7 +305,7 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
         enum TokenType
         {
             VARIABLE, FUNCTION, LITERAL, OPERATOR, UNARY_OPERATOR,
-            OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM
+            OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM, MARKER
         }
         public String surface = "";
         public TokenType type;