handling marker tokens

gradle.properties

@@ -4,8 +4,8 @@ org.gradle.jvmargs=-Xmx1G
 # Fabric Properties
 	# check these on https://fabricmc.net/use
     # or better: https://modmuss50.me/fabric.html
-	minecraft_version=1.14.3
+	minecraft_version=1.14.4-pre1
-	yarn_mappings=1.14.3+build.1
+	yarn_mappings=1.14.4-pre1+build.1
 	loader_version=0.4.8+build.155
 
 # Mod Properties

src/main/java/carpet/script/CarpetScriptServer.java

@@ -146,6 +146,8 @@ public class CarpetScriptServer
         {
             setChatErrorSnooper(source);
             CarpetExpression ex = new CarpetExpression(module.getCode(), source, new BlockPos(0, 0, 0));
+            ex.getExpr().asAModule();
+
             ex.scriptRunCommand(newHost, new BlockPos(source.getPosition()));
         }
         catch (CarpetExpressionException e)

src/main/java/carpet/script/Expression.java

@@ -335,6 +335,15 @@ public class Expression implements Cloneable
     private String expression;
     String getCodeString() {return expression;}
 
+    private boolean allowNewlineSubstitutions = true;
+    private boolean allowComments = false;
+
+    void asAModule()
+    {
+        allowNewlineSubstitutions = false;
+        allowComments = true;
+    }
+
     private String name;
     String getName() {return name;}
     Expression withName(String alias)
@@ -2578,8 +2587,8 @@ public class Expression implements Cloneable
      */
     public Expression(String expression)
     {
-        expression = expression.trim().replaceAll(";+$", "");
+        this.expression = expression.trim().replaceAll(";+$", "");
-        this.expression = expression.replaceAll("\\$", "\n").trim();
+        //this.expression = expression.replaceAll("\\$", "\n").trim();
         VariablesAndConstants();
         UserDefinedFunctionsAndControlFlow();
         Operators();
@@ -2588,12 +2597,12 @@ public class Expression implements Cloneable
         ListsLoopsAndHigherOrderFunctions();
     }
 
-    private List<Tokenizer.Token> shuntingYard(String expression)
+    private List<Tokenizer.Token> shuntingYard()
     {
         List<Tokenizer.Token> outputQueue = new ArrayList<>();
         Stack<Tokenizer.Token> stack = new Stack<>();
 
-        Tokenizer tokenizer = new Tokenizer(this, expression);
+        Tokenizer tokenizer = new Tokenizer(this, expression, allowComments, allowNewlineSubstitutions);
 
         Tokenizer.Token lastFunction = null;
         Tokenizer.Token previousToken = null;
@@ -2726,8 +2735,16 @@ public class Expression implements Cloneable
                     {
                         outputQueue.add(stack.pop());
                     }
+                case MARKER:
+                    if ("$".equals(token.surface))
+                    {
+                        StringBuilder sb = new StringBuilder(expression);
+                        sb.setCharAt(token.pos, '\n');
+                        expression = sb.toString();
+                    }
+                    break;
             }
-            previousToken = token;
+            if (token.type != Tokenizer.Token.TokenType.MARKER) previousToken = token;
         }
 
         while (!stack.isEmpty())
@@ -2796,7 +2813,7 @@ public class Expression implements Cloneable
     private LazyValue getAST()
     {
         Stack<LazyValue> stack = new Stack<>();
-        List<Tokenizer.Token> rpn = shuntingYard(this.expression);
+        List<Tokenizer.Token> rpn = shuntingYard();
         validate(rpn);
         for (final Tokenizer.Token token : rpn)
         {

src/main/java/carpet/script/ScriptHost.java

@@ -129,6 +129,7 @@ public class ScriptHost
                 case OPEN_PAREN:
                 case COMMA:
                 case CLOSE_PAREN:
+                case MARKER:
                     return "Fail: "+tok.surface+" is not allowed in invoke";
             }
         }

src/main/java/carpet/script/Tokenizer.java

@@ -67,7 +67,7 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
 
     public static List<Token> simplepass(String input)
     {
-        Tokenizer tok = new Tokenizer(null, input);
+        Tokenizer tok = new Tokenizer(null, input, false, false);
         List<Token> res = new ArrayList<>();
         while (tok.hasNext()) res.add(tok.next());
         return res;
@@ -233,6 +233,9 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
                         lineno++;
                         linepos = 0;
                     }
+                    token.append(greedyMatch);
+                    token.type = Token.TokenType.MARKER;
+                    return token; // skipping setting previous
                 }
                 pos++;
                 linepos++;