diff --git a/src/main/java/carpet/script/Tokenizer.java b/src/main/java/carpet/script/Tokenizer.java index f9196e21..bb97283b 100644 --- a/src/main/java/carpet/script/Tokenizer.java +++ b/src/main/java/carpet/script/Tokenizer.java @@ -22,6 +22,8 @@ public class Tokenizer implements Iterator private int pos = 0; private int lineno = 0; private int linepos = 0; + private boolean comments; + private boolean newLinesMarkers; /** The original input expression. */ @@ -31,10 +33,12 @@ public class Tokenizer implements Iterator private Expression expression; - Tokenizer(Expression expr, String input) + Tokenizer(Expression expr, String input, boolean allowComments, boolean allowNewLineMakers) { this.input = input; this.expression = expr; + this.comments = allowComments; + this.newLinesMarkers = allowNewLineMakers; } @@ -214,6 +218,22 @@ public class Tokenizer implements Iterator && (pos < input.length())) { greedyMatch += ch; + if (comments && "//".equals(greedyMatch)) + { + + while ( ch != '\n' && pos < input.length()) + { + ch = input.charAt(pos++); + linepos++; + greedyMatch += ch; + + } + if (ch=='\n') + { + lineno++; + linepos = 0; + } + } pos++; linepos++; if (Expression.none.isAnOperator(greedyMatch)) @@ -222,6 +242,14 @@ public class Tokenizer implements Iterator } ch = pos == input.length() ? 0 : input.charAt(pos); } + if (newLinesMarkers && "$".equals(greedyMatch)) + { + lineno++; + linepos = 0; + token.type = Token.TokenType.MARKER; + token.append('$'); + return token; // skipping previous token lookback + } if (validOperatorSeenUntil != -1) { token.append(input.substring(initialPos, validOperatorSeenUntil)); @@ -277,7 +305,7 @@ public class Tokenizer implements Iterator enum TokenType { VARIABLE, FUNCTION, LITERAL, OPERATOR, UNARY_OPERATOR, - OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM + OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM, MARKER } public String surface = ""; public TokenType type;