streamline parsing and evaluation structure

src/main/java/carpet/script/CarpetScriptHost.java

@@ -754,7 +754,7 @@ public class CarpetScriptHost extends ScriptHost
             }
         }
         String sign = "";
-        for (Token tok : Tokenizer.simplepass(arg))
+        for (Token tok : Tokenizer.simple(arg).parseTokens())
         {
             switch (tok.type)
             {

src/main/java/carpet/script/Expression.java

@@ -860,18 +860,13 @@ public class Expression
     }
 
 
-    private List<Token> shuntingYard(Context c)
+    private List<Token> shuntingYard(Context c, List<Token> tokens)
     {
         List<Token> outputQueue = new ArrayList<>();
         Stack<Token> stack = new ObjectArrayList<>();
-
-        Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions);
-        // stripping lousy but acceptable semicolons
-        List<Token> cleanedTokens = tokenizer.postProcess();
-
         Token lastFunction = null;
         Token previousToken = null;
-        for (Token token : cleanedTokens)
+        for (Token token : tokens)
         {
             switch (token.type)
             {
@@ -1188,7 +1183,11 @@ public class Expression
 
     private LazyValue getAST(Context context)
     {
-        List<Token> rpn = shuntingYard(context);
+        Tokenizer tokenizer = new Tokenizer(context, this, expression, allowComments, allowNewlineSubstitutions);
+        // stripping lousy but acceptable semicolons
+        List<Token> cleanedTokens = Tokenizer.postProcess(tokenizer.parseTokens());
+
+        List<Token> rpn = shuntingYard(context, cleanedTokens);
         validate(context, rpn);
         ExpressionNode root = RPNToParseTree(rpn, context);
         if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server))
@@ -1197,6 +1196,11 @@ public class Expression
         }
 
         Context optimizeOnlyContext = new Context.ContextForErrorReporting(context);
+        optimizeTree(context, root, optimizeOnlyContext);
+        return extractOp(optimizeOnlyContext, root, Context.Type.NONE);
+    }
+
+    private void optimizeTree(Context context, ExpressionNode root, Context optimizeOnlyContext) {
         boolean scriptsDebugging = Vanilla.ScriptServer_scriptDebugging(((CarpetScriptServer) context.scriptServer()).server);
         if (scriptsDebugging)
         {
@@ -1236,7 +1240,7 @@ public class Expression
                     prevTreeSize = treeSize(root);
                     prevTreeDepth = treeDepth(root);
                 }
-                boolean optimized = optimizeTree(optimizeOnlyContext, root, Context.Type.NONE, 0, scriptsDebugging);
+                boolean optimized = optimizeConstantsAndPureFunctions(optimizeOnlyContext, root, Context.Type.NONE, 0, scriptsDebugging);
                 if (!optimized)
                 {
                     break;
@@ -1248,7 +1252,6 @@ public class Expression
                 }
             }
         }
-        return extractOp(optimizeOnlyContext, root, Context.Type.NONE);
     }
 
     public Value explain(Context context)
@@ -1378,7 +1381,7 @@ public class Expression
         return optimized;
     }
 
-    private boolean optimizeTree(Context ctx, ExpressionNode node, Context.Type expectedType, int indent, boolean scriptsDebugging)
+    private boolean optimizeConstantsAndPureFunctions(Context ctx, ExpressionNode node, Context.Type expectedType, int indent, boolean scriptsDebugging)
     {
         // ctx is just to report errors, not values evaluation
         boolean optimized = false;
@@ -1400,7 +1403,7 @@ public class Expression
         Context.Type requestedType = operation.staticType(expectedType);
         for (ExpressionNode arg : node.args)
         {
-            if (optimizeTree(ctx, arg, requestedType, indent + 1, scriptsDebugging))
+            if (optimizeConstantsAndPureFunctions(ctx, arg, requestedType, indent + 1, scriptsDebugging))
             {
                 optimized = true;
             }

src/main/java/carpet/script/Tokenizer.java

@@ -1,20 +1,16 @@
 package carpet.script;
 
 import carpet.script.exception.ExpressionException;
-import carpet.script.exception.InternalExpressionException;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 /**
  * Expression tokenizer that allows to iterate over a {@link String}
  * expression token by token. Blank characters will be skipped.
  */
-public class Tokenizer implements Iterator<Token>
+public class Tokenizer
 {
     /**
      * What character to use for decimal separators.
@@ -53,15 +49,18 @@ public class Tokenizer implements Iterator<Token>
         this.newLinesMarkers = allowNewLineMakers;
     }
 
-    public List<Token> postProcess()
+    public static Tokenizer simple(String input)
+    {
+        return new Tokenizer(null, null, input, false, false);
+    }
+
+    public static List<Token> postProcess(List<Token> originalTokens)
     {
-        Iterable<Token> iterable = () -> this;
-        List<Token> originalTokens = StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList());
         List<Token> cleanedTokens = new ArrayList<>();
         Token last = null;
-        while (!originalTokens.isEmpty())
+        for (int i = originalTokens.size() - 1; i >= 0; i--)
         {
-            Token current = originalTokens.remove(originalTokens.size() - 1);
+            Token current = originalTokens.get(i);
             if (current.type == Token.TokenType.MARKER && current.surface.startsWith("//"))
             {
                 continue;
@@ -104,7 +103,16 @@ public class Tokenizer implements Iterator<Token>
         return cleanedTokens;
     }
 
-    @Override
+    public List<Token> parseTokens()
+    {
+        List<Token> tokens = new ArrayList<>();
+        while (hasNext())
+        {
+            tokens.add(next());
+        }
+        return tokens;
+    }
+
     public boolean hasNext()
     {
         return (pos < input.length());
@@ -132,18 +140,6 @@ public class Tokenizer implements Iterator<Token>
                 || (tok.type == Token.TokenType.UNARY_OPERATOR && tok.surface.equals(";u"));
     }
 
-    public static List<Token> simplepass(String input)
-    {
-        Tokenizer tok = new Tokenizer(null, null, input, false, false);
-        List<Token> res = new ArrayList<>();
-        while (tok.hasNext())
-        {
-            res.add(tok.next());
-        }
-        return res;
-    }
-
-    @Override
     public Token next()
     {
         Token token = new Token();
@@ -414,11 +410,4 @@ public class Tokenizer implements Iterator<Token>
         }
         return previousToken = token;
     }
-
-    @Override
-    public void remove()
-    {
-        throw new InternalExpressionException("remove() not supported");
-    }
-
 }