diff --git a/src/main/java/carpet/script/CarpetScriptHost.java b/src/main/java/carpet/script/CarpetScriptHost.java index a4c05d6c..7d6e18a6 100644 --- a/src/main/java/carpet/script/CarpetScriptHost.java +++ b/src/main/java/carpet/script/CarpetScriptHost.java @@ -754,7 +754,7 @@ public class CarpetScriptHost extends ScriptHost } } String sign = ""; - for (Token tok : Tokenizer.simplepass(arg)) + for (Token tok : Tokenizer.simple(arg).parseTokens()) { switch (tok.type) { diff --git a/src/main/java/carpet/script/Expression.java b/src/main/java/carpet/script/Expression.java index 90f7f0bc..ba02535d 100644 --- a/src/main/java/carpet/script/Expression.java +++ b/src/main/java/carpet/script/Expression.java @@ -860,18 +860,13 @@ public class Expression } - private List shuntingYard(Context c) + private List shuntingYard(Context c, List tokens) { List outputQueue = new ArrayList<>(); Stack stack = new ObjectArrayList<>(); - - Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions); - // stripping lousy but acceptable semicolons - List cleanedTokens = tokenizer.postProcess(); - Token lastFunction = null; Token previousToken = null; - for (Token token : cleanedTokens) + for (Token token : tokens) { switch (token.type) { @@ -1188,7 +1183,11 @@ public class Expression private LazyValue getAST(Context context) { - List rpn = shuntingYard(context); + Tokenizer tokenizer = new Tokenizer(context, this, expression, allowComments, allowNewlineSubstitutions); + // stripping lousy but acceptable semicolons + List cleanedTokens = Tokenizer.postProcess(tokenizer.parseTokens()); + + List rpn = shuntingYard(context, cleanedTokens); validate(context, rpn); ExpressionNode root = RPNToParseTree(rpn, context); if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server)) @@ -1197,7 +1196,12 @@ public class Expression } Context optimizeOnlyContext = new Context.ContextForErrorReporting(context); - boolean scriptsDebugging = Vanilla.ScriptServer_scriptDebugging(((CarpetScriptServer)context.scriptServer()).server); + optimizeTree(context, root, optimizeOnlyContext); + return extractOp(optimizeOnlyContext, root, Context.Type.NONE); + } + + private void optimizeTree(Context context, ExpressionNode root, Context optimizeOnlyContext) { + boolean scriptsDebugging = Vanilla.ScriptServer_scriptDebugging(((CarpetScriptServer) context.scriptServer()).server); if (scriptsDebugging) { CarpetScriptServer.LOG.info("Input code size for " + getModuleName() + ": " + treeSize(root) + " nodes, " + treeDepth(root) + " deep"); @@ -1236,7 +1240,7 @@ public class Expression prevTreeSize = treeSize(root); prevTreeDepth = treeDepth(root); } - boolean optimized = optimizeTree(optimizeOnlyContext, root, Context.Type.NONE, 0, scriptsDebugging); + boolean optimized = optimizeConstantsAndPureFunctions(optimizeOnlyContext, root, Context.Type.NONE, 0, scriptsDebugging); if (!optimized) { break; @@ -1248,7 +1252,6 @@ public class Expression } } } - return extractOp(optimizeOnlyContext, root, Context.Type.NONE); } public Value explain(Context context) @@ -1378,7 +1381,7 @@ public class Expression return optimized; } - private boolean optimizeTree(Context ctx, ExpressionNode node, Context.Type expectedType, int indent, boolean scriptsDebugging) + private boolean optimizeConstantsAndPureFunctions(Context ctx, ExpressionNode node, Context.Type expectedType, int indent, boolean scriptsDebugging) { // ctx is just to report errors, not values evaluation boolean optimized = false; @@ -1400,7 +1403,7 @@ public class Expression Context.Type requestedType = operation.staticType(expectedType); for (ExpressionNode arg : node.args) { - if (optimizeTree(ctx, arg, requestedType, indent + 1, scriptsDebugging)) + if (optimizeConstantsAndPureFunctions(ctx, arg, requestedType, indent + 1, scriptsDebugging)) { optimized = true; } diff --git a/src/main/java/carpet/script/Tokenizer.java b/src/main/java/carpet/script/Tokenizer.java index 21917031..e342022b 100644 --- a/src/main/java/carpet/script/Tokenizer.java +++ b/src/main/java/carpet/script/Tokenizer.java @@ -1,20 +1,16 @@ package carpet.script; import carpet.script.exception.ExpressionException; -import carpet.script.exception.InternalExpressionException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * Expression tokenizer that allows to iterate over a {@link String} * expression token by token. Blank characters will be skipped. */ -public class Tokenizer implements Iterator +public class Tokenizer { /** * What character to use for decimal separators. @@ -53,15 +49,18 @@ public class Tokenizer implements Iterator this.newLinesMarkers = allowNewLineMakers; } - public List postProcess() + public static Tokenizer simple(String input) + { + return new Tokenizer(null, null, input, false, false); + } + + public static List postProcess(List originalTokens) { - Iterable iterable = () -> this; - List originalTokens = StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList()); List cleanedTokens = new ArrayList<>(); Token last = null; - while (!originalTokens.isEmpty()) + for (int i = originalTokens.size() - 1; i >= 0; i--) { - Token current = originalTokens.remove(originalTokens.size() - 1); + Token current = originalTokens.get(i); if (current.type == Token.TokenType.MARKER && current.surface.startsWith("//")) { continue; @@ -104,7 +103,16 @@ public class Tokenizer implements Iterator return cleanedTokens; } - @Override + public List parseTokens() + { + List tokens = new ArrayList<>(); + while (hasNext()) + { + tokens.add(next()); + } + return tokens; + } + public boolean hasNext() { return (pos < input.length()); @@ -132,18 +140,6 @@ public class Tokenizer implements Iterator || (tok.type == Token.TokenType.UNARY_OPERATOR && tok.surface.equals(";u")); } - public static List simplepass(String input) - { - Tokenizer tok = new Tokenizer(null, null, input, false, false); - List res = new ArrayList<>(); - while (tok.hasNext()) - { - res.add(tok.next()); - } - return res; - } - - @Override public Token next() { Token token = new Token(); @@ -414,11 +410,4 @@ public class Tokenizer implements Iterator } return previousToken = token; } - - @Override - public void remove() - { - throw new InternalExpressionException("remove() not supported"); - } - }