Merge pull request #45 from gnembon/comments

Comments

gradle.properties

@@ -4,8 +4,8 @@ org.gradle.jvmargs=-Xmx1G
 # Fabric Properties
 	# check these on https://fabricmc.net/use
     # or better: https://modmuss50.me/fabric.html
-	minecraft_version=1.14.3
-	yarn_mappings=1.14.3+build.1
+	minecraft_version=1.14.4-pre1
+	yarn_mappings=1.14.4-pre1+build.1
 	loader_version=0.4.8+build.155
 
 # Mod Properties

src/main/java/carpet/script/CarpetScriptServer.java

@@ -146,6 +146,8 @@ public class CarpetScriptServer
         {
             setChatErrorSnooper(source);
             CarpetExpression ex = new CarpetExpression(module.getCode(), source, new BlockPos(0, 0, 0));
+            ex.getExpr().asAModule();
+
             ex.scriptRunCommand(newHost, new BlockPos(source.getPosition()));
         }
         catch (CarpetExpressionException e)

src/main/java/carpet/script/Expression.java

@@ -335,6 +335,15 @@ public class Expression implements Cloneable
     private String expression;
     String getCodeString() {return expression;}
 
+    private boolean allowNewlineSubstitutions = true;
+    private boolean allowComments = false;
+
+    void asAModule()
+    {
+        allowNewlineSubstitutions = false;
+        allowComments = true;
+    }
+
     private String name;
     String getName() {return name;}
     Expression withName(String alias)
@@ -2578,8 +2587,10 @@ public class Expression implements Cloneable
      */
     public Expression(String expression)
     {
-        expression = expression.trim().replaceAll(";+$", "");
-        this.expression = expression.replaceAll("\\$", "\n").trim();
+        this.expression = expression.trim().
+                replaceAll("\\r\\n?", "\n").
+                replaceAll(";+$", "");
+        //this.expression = expression.replaceAll("\\$", "\n").trim();
         VariablesAndConstants();
         UserDefinedFunctionsAndControlFlow();
         Operators();
@@ -2588,12 +2599,12 @@ public class Expression implements Cloneable
         ListsLoopsAndHigherOrderFunctions();
     }
 
-    private List<Tokenizer.Token> shuntingYard(String expression)
+    private List<Tokenizer.Token> shuntingYard()
     {
         List<Tokenizer.Token> outputQueue = new ArrayList<>();
         Stack<Tokenizer.Token> stack = new Stack<>();
 
-        Tokenizer tokenizer = new Tokenizer(this, expression);
+        Tokenizer tokenizer = new Tokenizer(this, expression, allowComments, allowNewlineSubstitutions);
 
         Tokenizer.Token lastFunction = null;
         Tokenizer.Token previousToken = null;
@@ -2726,8 +2737,16 @@ public class Expression implements Cloneable
                     {
                         outputQueue.add(stack.pop());
                     }
+                case MARKER:
+                    if ("$".equals(token.surface))
+                    {
+                        StringBuilder sb = new StringBuilder(expression);
+                        sb.setCharAt(token.pos, '\n');
+                        expression = sb.toString();
+                    }
+                    break;
             }
-            previousToken = token;
+            if (token.type != Tokenizer.Token.TokenType.MARKER) previousToken = token;
         }
 
         while (!stack.isEmpty())
@@ -2796,7 +2815,7 @@ public class Expression implements Cloneable
     private LazyValue getAST()
     {
         Stack<LazyValue> stack = new Stack<>();
-        List<Tokenizer.Token> rpn = shuntingYard(this.expression);
+        List<Tokenizer.Token> rpn = shuntingYard();
         validate(rpn);
         for (final Tokenizer.Token token : rpn)
         {

src/main/java/carpet/script/ScriptHost.java

@@ -129,6 +129,7 @@ public class ScriptHost
                 case OPEN_PAREN:
                 case COMMA:
                 case CLOSE_PAREN:
+                case MARKER:
                     return "Fail: "+tok.surface+" is not allowed in invoke";
             }
         }

src/main/java/carpet/script/Tokenizer.java

@@ -22,6 +22,8 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
     private int pos = 0;
     private int lineno = 0;
     private int linepos = 0;
+    private boolean comments;
+    private boolean newLinesMarkers;
 
 
     /** The original input expression. */
@@ -31,10 +33,12 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
 
     private Expression expression;
 
-    Tokenizer(Expression expr, String input)
+    Tokenizer(Expression expr, String input, boolean allowComments, boolean allowNewLineMakers)
     {
         this.input = input;
         this.expression = expr;
+        this.comments = allowComments;
+        this.newLinesMarkers = allowNewLineMakers;
     }
 
 
@@ -63,7 +67,7 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
 
     public static List<Token> simplepass(String input)
     {
-        Tokenizer tok = new Tokenizer(null, input);
+        Tokenizer tok = new Tokenizer(null, input, false, false);
         List<Token> res = new ArrayList<>();
         while (tok.hasNext()) res.add(tok.next());
         return res;
@@ -214,6 +218,25 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
                     && (pos < input.length()))
             {
                 greedyMatch += ch;
+                if (comments && "//".equals(greedyMatch))
+                {
+
+                    while ( ch != '\n' && pos < input.length())
+                    {
+                        ch = input.charAt(pos++);
+                        linepos++;
+                        greedyMatch += ch;
+
+                    }
+                    if (ch=='\n')
+                    {
+                        lineno++;
+                        linepos = 0;
+                    }
+                    token.append(greedyMatch);
+                    token.type = Token.TokenType.MARKER;
+                    return token; // skipping setting previous
+                }
                 pos++;
                 linepos++;
                 if (Expression.none.isAnOperator(greedyMatch))
@@ -222,6 +245,14 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
                 }
                 ch = pos == input.length() ? 0 : input.charAt(pos);
             }
+            if (newLinesMarkers && "$".equals(greedyMatch))
+            {
+                lineno++;
+                linepos = 0;
+                token.type = Token.TokenType.MARKER;
+                token.append('$');
+                return token; // skipping previous token lookback
+            }
             if (validOperatorSeenUntil != -1)
             {
                 token.append(input.substring(initialPos, validOperatorSeenUntil));
@@ -277,7 +308,7 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
         enum TokenType
         {
             VARIABLE, FUNCTION, LITERAL, OPERATOR, UNARY_OPERATOR,
-            OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM
+            OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM, MARKER
         }
         public String surface = "";
         public TokenType type;