improved explain, still hidden

src/main/java/carpet/script/CarpetExpression.java

@@ -20,6 +20,7 @@ import carpet.script.value.Value;
 import net.minecraft.commands.CommandSourceStack;
 import net.minecraft.core.BlockPos;
 import net.minecraft.world.entity.Entity;
+import org.apache.commons.lang3.tuple.Pair;
 
 import javax.annotation.Nullable;
 import java.util.List;
@@ -93,8 +94,9 @@ public class CarpetExpression
             return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.executeAndEvaluate(
                     context,
                     Vanilla.ScriptServer_scriptOptimizations(scriptServer.server),
+                    host.loadOverrides,
                     Vanilla.ScriptServer_scriptDebugging(scriptServer.server) ? CarpetScriptServer.LOG::info : null
-            ).getBoolean());
+            ).getLeft().getBoolean());
         }
         catch (ExpressionException e)
         {
@@ -110,7 +112,7 @@ public class CarpetExpression
         }
     }
 
-    public Value scriptRunCommand(ScriptHost host, BlockPos pos)
+    public Pair<Value, Expression.ExpressionNode> scriptRunCommand(ScriptHost host, BlockPos pos)
     {
         CarpetScriptServer scriptServer = (CarpetScriptServer) host.scriptServer();
         if (scriptServer.stopAll)
@@ -137,6 +139,7 @@ public class CarpetExpression
             return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.executeAndEvaluate(
                     context,
                     Vanilla.ScriptServer_scriptOptimizations(scriptServer.server),
+                    host.loadOverrides,
                     Vanilla.ScriptServer_scriptDebugging(scriptServer.server) ? CarpetScriptServer.LOG::info : null
             ));
         }
@@ -154,7 +157,7 @@ public class CarpetExpression
         }
     }
 
-    public List<Token> explain(ScriptHost host, @Nullable String method, @Nullable String style, BlockPos pos)
+    public List<Token> explain(ScriptHost host, @Nullable String code, @Nullable String method, @Nullable String style, BlockPos pos)
     {
         CarpetScriptServer scriptServer = (CarpetScriptServer) host.scriptServer();
         try
@@ -174,7 +177,7 @@ public class CarpetExpression
                 Value playerValue = new EntityValue(e).bindTo("p");
                 context.with("p", (cc, tt) -> playerValue);
             }
-            return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.explain(context, method, style));
+            return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.explain(context, code, method, style));
         }
         catch (ExpressionException e)
         {

src/main/java/carpet/script/CarpetScriptHost.java

@@ -78,9 +78,9 @@ public class CarpetScriptHost extends ScriptHost
     public AppStoreManager.StoreNode storeSource;
     boolean hasCommand;
 
-    private CarpetScriptHost(CarpetScriptServer server, @Nullable Module code, boolean perUser, ScriptHost parent, Map<Value, Value> config, Map<String, CommandArgument> argTypes, Predicate<CommandSourceStack> commandValidator, boolean isRuleApp)
+    private CarpetScriptHost(CarpetScriptServer server, @Nullable Module code, boolean perUser, ScriptHost parent, Map<Value, Value> config, Map<String, CommandArgument> argTypes, Predicate<CommandSourceStack> commandValidator, boolean isRuleApp, Expression.LoadOverride override)
     {
-        super(code, server, perUser, parent);
+        super(code, server, perUser, parent, override);
         this.saveTimeout = 0;
         persistenceRequired = true;
         if (parent == null && code != null) // app, not a global host
@@ -99,9 +99,9 @@ public class CarpetScriptHost extends ScriptHost
         storeSource = null;
     }
 
-    public static CarpetScriptHost create(CarpetScriptServer scriptServer, @Nullable Module module, boolean perPlayer, CommandSourceStack source, Predicate<CommandSourceStack> commandValidator, boolean isRuleApp, AppStoreManager.StoreNode storeSource)
+    public static CarpetScriptHost create(CarpetScriptServer scriptServer, @Nullable Module module, boolean perPlayer, CommandSourceStack source, Predicate<CommandSourceStack> commandValidator, boolean isRuleApp, AppStoreManager.StoreNode storeSource, Expression.LoadOverride override)
     {
-        CarpetScriptHost host = new CarpetScriptHost(scriptServer, module, perPlayer, null, Collections.emptyMap(), new HashMap<>(), commandValidator, isRuleApp);
+        CarpetScriptHost host = new CarpetScriptHost(scriptServer, module, perPlayer, null, Collections.emptyMap(), new HashMap<>(), commandValidator, isRuleApp, override);
         // parse code and convert to expression
         if (module != null)
         {
@@ -111,7 +111,7 @@ public class CarpetScriptHost extends ScriptHost
                 CarpetExpression ex = new CarpetExpression(host.main, module.code(), source, new BlockPos(0, 0, 0));
                 ex.getExpr().asATextSource();
                 host.storeSource = storeSource;
-                ex.scriptRunCommand(host, BlockPos.containing(source.getPosition()));
+                host.root = ex.scriptRunCommand(host, BlockPos.containing(source.getPosition())).getRight();
             }
             catch (CarpetExpressionException e)
             {
@@ -132,6 +132,10 @@ public class CarpetScriptHost extends ScriptHost
                 host.storeSource = null;
             }
         }
+        else
+        {
+            host.root = Expression.ExpressionNode.ofConstant(Value.NULL, new Token());
+        }
         return host;
     }
 
@@ -253,7 +257,7 @@ public class CarpetScriptHost extends ScriptHost
     @Override
     protected ScriptHost duplicate()
     {
-        return new CarpetScriptHost(scriptServer(), main, false, this, appConfig, appArgTypes, commandValidator, isRuleApp);
+        return new CarpetScriptHost(scriptServer(), main, false, this, appConfig, appArgTypes, commandValidator, isRuleApp, loadOverrides);
     }
 
     @Override

src/main/java/carpet/script/CarpetScriptServer.java

@@ -121,7 +121,7 @@ public class CarpetScriptServer extends ScriptServer
         tickStart = 0L;
         stopAll = false;
         holyMoly = server.getCommands().getDispatcher().getRoot().getChildren().stream().map(CommandNode::getName).collect(Collectors.toSet());
-        globalHost = CarpetScriptHost.create(this, null, false, null, p -> true, false, null);
+        globalHost = CarpetScriptHost.create(this, null, false, null, p -> true, false, null, Expression.LoadOverride.DEFAULT);
     }
 
     public void initializeForWorld()
@@ -130,7 +130,7 @@ public class CarpetScriptServer extends ScriptServer
         {
             for (String moduleName : listAvailableModules(false))
             {
-                addScriptHost(server.createCommandSourceStack(), moduleName, null, true, true, false, null);
+                addScriptHost(server.createCommandSourceStack(), moduleName, null, true, true, false, null, Expression.LoadOverride.DEFAULT);
             }
         }
         CarpetEventServer.Event.START.onTick(server);
@@ -232,8 +232,8 @@ public class CarpetScriptServer extends ScriptServer
         return name == null ? globalHost : modules.get(name);
     }
 
-    public boolean addScriptHost(CommandSourceStack source, String name, @Nullable Predicate<CommandSourceStack> commandValidator,
-                                 boolean perPlayer, boolean autoload, boolean isRuleApp, AppStoreManager.StoreNode installer)
+    public int addScriptHost(CommandSourceStack source, String name, @Nullable Predicate<CommandSourceStack> commandValidator,
+                                 boolean perPlayer, boolean autoload, boolean isRuleApp, AppStoreManager.StoreNode installer, Expression.LoadOverride override)
     {
         Runnable token = Carpet.startProfilerSection("Scarpet load");
         if (commandValidator == null)
@@ -247,7 +247,7 @@ public class CarpetScriptServer extends ScriptServer
         {
             if (isRuleApp)
             {
-                return false;
+                return 0;
             }
             removeScriptHost(source, name, false, isRuleApp);
             reload = true;
@@ -256,17 +256,17 @@ public class CarpetScriptServer extends ScriptServer
         if (module == null)
         {
             Carpet.Messenger_message(source, "r Failed to add " + name + " app: App not found");
-            return false;
+            return 0;
         }
         CarpetScriptHost newHost;
         try
         {
-            newHost = CarpetScriptHost.create(this, module, perPlayer, source, commandValidator, isRuleApp, installer);
+            newHost = CarpetScriptHost.create(this, module, perPlayer, source, commandValidator, isRuleApp, installer, override);
         }
         catch (LoadException e)
         {
             Carpet.Messenger_message(source, "r Failed to add " + name + " app" + (e.getMessage() == null ? "" : ": " + e.getMessage()));
-            return false;
+            return 0;
         }
 
         modules.put(name, newHost);
@@ -278,7 +278,7 @@ public class CarpetScriptServer extends ScriptServer
         if (autoload && !newHost.persistenceRequired)
         {
             removeScriptHost(source, name, false, false);
-            return false;
+            return 0;
         }
         String action = (installer != null) ? (reload ? "reinstalled" : "installed") : (reload ? "reloaded" : "loaded");
 
@@ -292,7 +292,7 @@ public class CarpetScriptServer extends ScriptServer
         if (isCommandAdded == null) // error should be dispatched
         {
             removeScriptHost(source, name, false, isRuleApp);
-            return false;
+            return 0;
         }
         else if (isCommandAdded)
         {
@@ -330,7 +330,7 @@ public class CarpetScriptServer extends ScriptServer
         token.run();
         long end = System.nanoTime();
         LOG.info("App " + name + " loaded in " + (end - start) / 1000000 + " ms");
-        return true;
+        return 1;
     }
 
     public boolean isInvalidCommandRoot(String appName)
@@ -458,11 +458,11 @@ public class CarpetScriptServer extends ScriptServer
     }
 
     private record TransferData(boolean perUser, Predicate<CommandSourceStack> commandValidator,
-                                boolean isRuleApp)
+                                boolean isRuleApp, Expression.LoadOverride override)
     {
         private TransferData(CarpetScriptHost host)
         {
-            this(host.perUser, host.commandValidator, host.isRuleApp);
+            this(host.perUser, host.commandValidator, host.isRuleApp, host.loadOverrides);
         }
     }
 
@@ -473,7 +473,7 @@ public class CarpetScriptServer extends ScriptServer
         apps.keySet().forEach(s -> removeScriptHost(server.createCommandSourceStack(), s, false, false));
         CarpetEventServer.Event.clearAllBuiltinEvents();
         init();
-        apps.forEach((s, data) -> addScriptHost(server.createCommandSourceStack(), s, data.commandValidator, data.perUser, false, data.isRuleApp, null));
+        apps.forEach((s, data) -> addScriptHost(server.createCommandSourceStack(), s, data.commandValidator, data.perUser, false, data.isRuleApp, null, data.override));
     }
 
     public void reAddCommands()

src/main/java/carpet/script/Expression.java

@@ -36,6 +36,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 
 import javax.annotation.Nullable;
 import java.math.BigInteger;
@@ -95,6 +96,9 @@ public class Expression
     @Nullable
     private LazyValue ast = null;
 
+    @Nullable
+    private ExpressionNode root = null;
+
     /**
      * script specific operatos and built-in functions
      */
@@ -1070,7 +1074,7 @@ public class Expression
      */
     public Expression(String expression)
     {
-        this.expression = expression.stripTrailing().replaceAll("\\r\\n?", "\n").replaceAll("\\t", "   ");
+        this.expression = stripExpression(expression);
         Operators.apply(this);
         ControlFlow.apply(this);
         Functions.apply(this);
@@ -1084,6 +1088,11 @@ public class Expression
         }
     }
 
+    private String stripExpression(String expression)
+    {
+        return expression.stripTrailing().replaceAll("\\r\\n?", "\n").replaceAll("\\t", "   ");
+    }
+
 
     private List<Token> shuntingYard(Context c, List<Token> tokens)
     {
@@ -1238,13 +1247,29 @@ public class Expression
         }
     }
 
-    public Value executeAndEvaluate(Context c, boolean optimize, @Nullable Consumer<String> logger)
+    public enum LoadOverride {
+        DEFAULT("clean"), CANONICAL("canonical"), OPTIMIZED("optimized"), FUNCTIONAL("functional"), FUNCTIONAL_OPTIMIZED("functional_optimized");
+        public String equivalent;
+        LoadOverride(String equivalent) {
+            this.equivalent = equivalent;
+        }
+    }
+
+    public Pair<Value, ExpressionNode> executeAndEvaluate(Context c, boolean optimize, LoadOverride override, @Nullable Consumer<String> logger)
     {
         if (ast == null)
         {
-            ast = getAST(c, optimize, logger);
+            boolean functions = false;
+            if (override != LoadOverride.DEFAULT) {
+                optimize = override == LoadOverride.OPTIMIZED || override == LoadOverride.FUNCTIONAL_OPTIMIZED;
+                functions = override == LoadOverride.FUNCTIONAL || override == LoadOverride.FUNCTIONAL_OPTIMIZED;
+            }
+
+            Pair<ExpressionNode, LazyValue> ret = getAST(c, optimize, functions, logger);
+            ast = ret.getRight();
+            root = ret.getLeft();
         }
-        return evaluatePartial(() -> ast, c, Context.Type.NONE);
+        return Pair.of(evaluatePartial(() -> ast, c, Context.Type.NONE), root);
     }
 
     public Value evaluatePartial(Supplier<LazyValue> exprProvider, Context c, Context.Type expectedType)
@@ -1301,30 +1326,30 @@ public class Expression
             return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token);
         }
 
-        public List<Token> tokensRecursive(Expression expression) {
+        public List<Token> tokensRecursive(Expression expression, List<Token> reference, Map<Pair<Integer, Integer>, List<Integer>> referencePointers){
             List<Token> tokens = new ArrayList<>();
             switch (token.type) {
                 case FUNCTION -> {
                     tokens.add(token);
-                    tokens.add(token.morphedInto(Token.TokenType.OPEN_PAREN, "("));
+                    tokens.add(findToken(token, Token.TokenType.OPEN_PAREN, "(", reference, referencePointers));
                     if (!args.isEmpty()) {
                         for (ExpressionNode arg : args) {
-                            List<Token> argTokens = arg.tokensRecursive(expression);
+                            List<Token> argTokens = arg.tokensRecursive(expression, reference, referencePointers);
                             tokens.addAll(argTokens);
-                            tokens.add(argTokens.getLast().morphedInto(Token.TokenType.COMMA, ",").disguiseAs(", ", null));
+                            tokens.add(findToken(argTokens.getLast(), Token.TokenType.COMMA, ",", reference, referencePointers).disguiseAs(", ", null));
                         }
                         tokens.removeLast();
                     }
-                    tokens.add(tokens.getLast().morphedInto(Token.TokenType.CLOSE_PAREN, ")"));
+                    tokens.add(findToken(tokens.getLast(), Token.TokenType.CLOSE_PAREN, ")", reference, referencePointers));
                 }
                 case OPERATOR -> {
-                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0)));
-                    tokens.add(token.disguiseAs(" "+token.surface+" ", null));
-                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(1)));
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0), reference, referencePointers));
+                    tokens.add(token.disguiseAs(token.surface.equals(";") ? (token.surface+" ") : (" "+token.surface+" "), null));
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(1), reference, referencePointers));
                 }
                 case UNARY_OPERATOR -> {
                     tokens.add(token.disguiseAs(" "+StringUtils.chop(token.surface),null));
-                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0)));
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0), reference, referencePointers));
                 }
                 case VARIABLE, LITERAL, HEX_LITERAL, CONSTANT -> tokens.add(token);
                 case STRINGPARAM -> tokens.add(token.disguiseAs("'"+token.surface+"'", null));
@@ -1336,7 +1361,29 @@ public class Expression
             return tokens;
         }
 
-        public List<Token> createOperatorArgumentTokens(Expression expression, ExpressionNode operand) {
+        public Token findToken(Token previous, Token.TokenType type, String expectedSurface, List<Token> reference, Map<Pair<Integer, Integer>, List<Integer>> referencePointers)
+        {
+            List<Integer> indices = referencePointers.get(Pair.of(previous.lineno, previous.linepos));
+            if (indices == null)
+            {
+                return previous.morphedInto(type, expectedSurface);
+            }
+            for (int index : indices)
+            {
+                if (index + 1 == reference.size())
+                {
+                    continue;
+                }
+                Token token = reference.get(index + 1);
+                if (token.type == type)
+                {
+                    return token.morphedInto(type, expectedSurface);
+                }
+            }
+            return previous.morphedInto(type, expectedSurface);
+        }
+
+        public List<Token> createOperatorArgumentTokens(Expression expression, ExpressionNode operand, List<Token> reference, Map<Pair<Integer, Integer>, List<Integer>> referencePointers) {
             boolean needsBrackets = false;
             var operators = expression.operators;
             if (operators.containsKey(operand.token.surface) && operators.containsKey(token.surface)) {
@@ -1344,15 +1391,15 @@ public class Expression
                     needsBrackets = true;
                 }
             }
-            List<Token> argumentTokens = operand.tokensRecursive(expression);
+            List<Token> argumentTokens = operand.tokensRecursive(expression, reference, referencePointers);
             if (!needsBrackets) {
                 return argumentTokens;
             }
             List<Token> bracketed = new ArrayList<>();
-            bracketed.add(argumentTokens.getFirst().morphedInto(Token.TokenType.OPEN_PAREN, "("));
+            bracketed.add(findToken(argumentTokens.getFirst(), Token.TokenType.OPEN_PAREN, "(", reference, referencePointers));
             bracketed.get(0).swapPlace(argumentTokens.getFirst());
             bracketed.addAll(argumentTokens);
-            bracketed.add(argumentTokens.getLast().morphedInto(Token.TokenType.CLOSE_PAREN, ")"));
+            bracketed.add(findToken(argumentTokens.getLast(), Token.TokenType.CLOSE_PAREN, ")", reference, referencePointers));
             return bracketed;
         }
     }
@@ -1367,24 +1414,32 @@ public class Expression
                 case UNARY_OPERATOR -> {
                     ExpressionNode node = nodeStack.pop();
                     LazyValue result = (c, t) -> operators.get(token.surface).lazyEval(c, t, this, token, node.op, null).evalValue(c, t);
-                    nodeStack.push(new ExpressionNode(result, Collections.singletonList(node), token));
+                    ExpressionNode newNode = new ExpressionNode(result, Collections.singletonList(node), token);
+                    token.node = newNode;
+                    nodeStack.push(newNode);
                 }
                 case OPERATOR -> {
                     ExpressionNode v1 = nodeStack.pop();
                     ExpressionNode v2 = nodeStack.pop();
                     LazyValue result = (c, t) -> operators.get(token.surface).lazyEval(c, t, this, token, v2.op, v1.op).evalValue(c, t);
-                    nodeStack.push(new ExpressionNode(result, List.of(v2, v1), token));
+                    ExpressionNode newNode = new ExpressionNode(result, List.of(v2, v1), token);
+                    token.node = newNode;
+                    nodeStack.push(newNode);
                 }
                 case VARIABLE -> {
                     Value constant = getConstantFor(token.surface);
                     if (constant != null)
                     {
                         token.morph(Token.TokenType.CONSTANT, token.surface);
-                        nodeStack.push(new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token));
+                        ExpressionNode newNode = new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token);
+                        token.node = newNode;
+                        nodeStack.push(newNode);
                     }
                     else
                     {
-                        nodeStack.push(new ExpressionNode(((c, t) -> getOrSetAnyVariable(c, token.surface).evalValue(c, t)), Collections.emptyList(), token));
+                        ExpressionNode newNode = new ExpressionNode(((c, t) -> getOrSetAnyVariable(c, token.surface).evalValue(c, t)), Collections.emptyList(), token);
+                        token.node = newNode;
+                        nodeStack.push(newNode);
                     }
                 }
                 case FUNCTION -> {
@@ -1419,12 +1474,17 @@ public class Expression
                         nodeStack.pop();
                     }
                     List<LazyValue> params = p.stream().map(n -> n.op).collect(Collectors.toList());
-                    nodeStack.push(new ExpressionNode(
+                    ExpressionNode newNode = new ExpressionNode(
                             (c, t) -> f.lazyEval(c, t, this, token, params).evalValue(c, t),
                             p, token
-                    ));
+                    );
+                    token.node = newNode;
+                    nodeStack.push(newNode);
+                }
+                case OPEN_PAREN -> {
+                    token.node = ExpressionNode.PARAMS_START;
+                    nodeStack.push(ExpressionNode.PARAMS_START);
                 }
-                case OPEN_PAREN -> nodeStack.push(ExpressionNode.PARAMS_START);
                 case LITERAL -> {
                     Value number;
                     try
@@ -1436,11 +1496,15 @@ public class Expression
                         throw new ExpressionException(context, this, token, "Not a number");
                     }
                     //token.morph(Token.TokenType.CONSTANT, token.surface);
-                    nodeStack.push(ExpressionNode.ofConstant(number, token));
+                    ExpressionNode newNode = ExpressionNode.ofConstant(number, token);
+                    token.node = newNode;
+                    nodeStack.push(newNode);
                 }
                 case STRINGPARAM -> {
                     //token.morph(Token.TokenType.CONSTANT, token.surface);
-                    nodeStack.push(ExpressionNode.ofConstant(new StringValue(token.surface), token));
+                    ExpressionNode newNode = ExpressionNode.ofConstant(new StringValue(token.surface), token);
+                    token.node = newNode;
+                    nodeStack.push(newNode);
                 }
                 case HEX_LITERAL -> {
                     Value hexNumber;
@@ -1453,7 +1517,9 @@ public class Expression
                         throw new ExpressionException(context, this, token, "Not a number");
                     }
                     //token.morph(Token.TokenType.CONSTANT, token.surface);
-                    nodeStack.push(ExpressionNode.ofConstant(hexNumber, token));
+                    ExpressionNode newNode = ExpressionNode.ofConstant(hexNumber, token);
+                    token.node = newNode;
+                    nodeStack.push(newNode);
                 }
                 default -> throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'");
             }
@@ -1461,7 +1527,7 @@ public class Expression
         return nodeStack.pop();
     }
 
-    private LazyValue getAST(Context context, boolean optimize, @Nullable Consumer<String> logger)
+    private Pair<ExpressionNode, LazyValue> getAST(Context context, boolean optimize, boolean functional, @Nullable Consumer<String> logger)
     {
         Tokenizer tokenizer = new Tokenizer(context, this, expression, allowComments, allowNewlineSubstitutions);
         // stripping lousy but acceptable semicolons
@@ -1470,19 +1536,22 @@ public class Expression
         List<Token> rpn = shuntingYard(context, cleanedTokens);
         validate(context, rpn);
         ExpressionNode root = RPNToParseTree(rpn, context);
-        if (!optimize)
+        if (!optimize && !functional)
         {
-            return root.op;
+            return Pair.of(root, root.op);
         }
 
         Context optimizeOnlyContext = new Context.ContextForErrorReporting(context);
         // flipping to full functional representation makes it little underperforming, might be related
         // to the fact that operators are running from a bigger pool or function execution is slower
-        optimizeTree(root, optimizeOnlyContext, logger, false);
-        return extractOp(optimizeOnlyContext, root, Context.Type.NONE);
+        optimizeTree(root, optimizeOnlyContext, logger, optimize, functional);
+        if (!optimize) {
+            return Pair.of(root, root.op);
+        }
+        return Pair.of(root, extractOp(optimizeOnlyContext, root, Context.Type.NONE));
     }
 
-    private void optimizeTree(ExpressionNode root, Context optimizeOnlyContext, @Nullable Consumer<String> logger, boolean toFunctional) {
+    private void optimizeTree(ExpressionNode root, Context optimizeOnlyContext, @Nullable Consumer<String> logger, boolean optimize, boolean toFunctional) {
         if (logger != null)
         {
             logger.accept("Input code size for " + getModuleName() + ": " + treeSize(root) + " nodes, " + treeDepth(root) + " deep");
@@ -1503,7 +1572,7 @@ public class Expression
                     prevTreeSize = treeSize(root);
                     prevTreeDepth = treeDepth(root);
                 }
-                boolean optimized = compactTree(root, Context.Type.NONE, 0, logger, toFunctional);
+                boolean optimized = compactTree(root, Context.Type.NONE, 0, logger, optimize, toFunctional);
                 if (!optimized)
                 {
                     break;
@@ -1514,7 +1583,7 @@ public class Expression
                     logger.accept("Compacted from " + prevTreeSize + " nodes, " + prevTreeDepth + " code depth to " + treeSize(root) + " nodes, " + treeDepth(root) + " code depth");
                 }
             }
-            while (true)
+            while (optimize)
             {
                 if (logger != null)
                 {
@@ -1535,10 +1604,48 @@ public class Expression
         }
     }
 
-    public List<Token> explain(Context context, @Nullable String method, @Nullable String style)
+    public List<Token> explain(Context context, @Nullable String code, @Nullable String method, @Nullable String style)
     {
+        if (code == null)
+        {
+            ExpressionNode node;
+            if (method != null)
+            {
+                FunctionValue function = context.host.getFunction(method);
+                if (function == null) {
+                    throw new ExpressionException(context, this, "Unknown function " + method);
+                }
+                node = function.getToken().node;
+                if (node == null) {
+                    throw new ExpressionException(context, this, "Function " + method + " is not a compiled function");
+                }
+            } else
+            {
+                node = context.host.root;
+            }
+            if (node == null || node.token.type == null) {
+                throw new ExpressionException(context, this, "No code to explain");
+            }
+
+            // grab source reference
+            Tokenizer tokenizer = new Tokenizer(context, this, context.host.main == null ? "" : stripExpression(context.host.main.code()) , true, true);
+            List<Token> input = tokenizer.parseTokens();
+            List<Token> cleanedTokens = Tokenizer.postProcess(input);
+
+            Map<Pair<Integer, Integer>, List<Integer>> tokenPointers = new HashMap<>();
+            for (int i = 0; i < cleanedTokens.size(); i++) {
+                Token token = cleanedTokens.get(i);
+                tokenPointers.computeIfAbsent(Pair.of(token.lineno, token.linepos), k -> new ArrayList<>()).add(i);
+            }
+
+            return node.tokensRecursive(this, cleanedTokens, tokenPointers);
+        }
+        if (style == null) {
+            style = context.host.loadOverrides.equivalent;
+        }
+
         //todo convert to explain
-        Tokenizer tokenizer = new Tokenizer(context, this, expression, true, true);
+        Tokenizer tokenizer = new Tokenizer(context, this, stripExpression(code), true, true);
         // stripping lousy but acceptable semicolons
         List<Token> input = tokenizer.parseTokens();
         if (style.equalsIgnoreCase("raw")) {
@@ -1552,15 +1659,22 @@ public class Expression
         List<Token> rpn = shuntingYard(context, cleanedTokens);
         validate(context, rpn);
         ExpressionNode root = RPNToParseTree(rpn, context);
-        List<Token> parsedTokens = root.tokensRecursive(this);
+
+        Map<Pair<Integer, Integer>, List<Integer>> tokenPointers = new HashMap<>();
+        for (int i = 0; i < cleanedTokens.size(); i++) {
+            Token token = cleanedTokens.get(i);
+            tokenPointers.computeIfAbsent(Pair.of(token.lineno, token.linepos), k -> new ArrayList<>()).add(i);
+        }
+
+        List<Token> parsedTokens = root.tokensRecursive(this, cleanedTokens, tokenPointers);
         if (style.equalsIgnoreCase("canonical")) {
             return parsedTokens;
         }
 
         Context optimizeOnlyContext = new Context.ContextForErrorReporting(context);
         // pure functional
-        optimizeTree(root, optimizeOnlyContext, null,  style.equalsIgnoreCase("functional"));
-        List<Token> compileTimeOptimized = root.tokensRecursive(this);
+        optimizeTree(root, optimizeOnlyContext, null, style.contains("optimized"), style.contains("functional"));
+        List<Token> compileTimeOptimized = root.tokensRecursive(this, cleanedTokens, tokenPointers);
 
         return compileTimeOptimized;
     }
@@ -1576,7 +1690,7 @@ public class Expression
     }
 
 
-    private boolean compactTree(ExpressionNode node, Context.Type expectedType, int indent, @Nullable Consumer<String> logger, boolean toFunctional)
+    private boolean compactTree(ExpressionNode node, Context.Type expectedType, int indent, @Nullable Consumer<String> logger, boolean optimize, boolean toFunctional)
     {
         // ctx is just to report errors, not values evaluation
         boolean optimized = false;
@@ -1596,13 +1710,13 @@ public class Expression
         Context.Type requestedType = operation.staticType(expectedType);
         for (ExpressionNode arg : node.args)
         {
-            if (compactTree(arg, requestedType, indent + 1, logger, toFunctional))
+            if (compactTree(arg, requestedType, indent + 1, logger, optimize, toFunctional))
             {
                 optimized = true;
             }
         }
 
-        if (expectedType != Context.Type.MAPDEF && symbol.equals("->") && node.args.size() == 2)
+        if (optimize && expectedType != Context.Type.MAPDEF && (symbol.equals("->") || symbol.equals("define") ) && node.args.size() == 2)
         {
             String rop = node.args.get(1).token.surface;
             ExpressionNode returnNode = null;
@@ -1648,7 +1762,7 @@ public class Expression
         {
             String operator = pair.getKey();
             String function = pair.getValue();
-            if ((symbol.equals(operator) || symbol.equals(function)) && !node.args.isEmpty())
+            if (optimize && (symbol.equals(operator) || symbol.equals(function)) && !node.args.isEmpty())
             {
                 boolean leftOptimizable = operators.get(operator).isLeftAssoc();
                 ExpressionNode optimizedChild = node.args.get(leftOptimizable ? 0 : (node.args.size() - 1));

src/main/java/carpet/script/ScriptCommand.java

@@ -294,18 +294,18 @@ public class ScriptCommand
         LiteralArgumentBuilder<CommandSourceStack> a = literal("load").requires(Vanilla::ServerPlayer_canScriptACE).
                 then(argument("app", StringArgumentType.word()).
                         suggests((cc, bb) -> suggest(ss(cc).listAvailableModules(true), bb)).
-                        executes((cc) ->
-                        {
-                            boolean success = ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null);
-                            return success ? 1 : 0;
-                        }).
+                        executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null, Expression.LoadOverride.DEFAULT)).
+                        //then(literal("canonical").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null, Expression.LoadOverride.CANONICAL))).
+                        //then(literal("optimized").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null, Expression.LoadOverride.OPTIMIZED))).
+                        //then(literal("functional").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null, Expression.LoadOverride.FUNCTIONAL))).
+                        //then(literal("functional_optimized").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, true, false, false, null, Expression.LoadOverride.FUNCTIONAL_OPTIMIZED))).
                         then(literal("global").
-                                executes((cc) ->
-                                        {
-                                            boolean success = ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null);
-                                            return success ? 1 : 0;
-                                        }
-                                )
+                                executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null, Expression.LoadOverride.DEFAULT))
+                                        //.
+                                //then(literal("canonical").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null, Expression.LoadOverride.CANONICAL))).
+                                //then(literal("optimized").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null, Expression.LoadOverride.OPTIMIZED))).
+                                //then(literal("functional").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null, Expression.LoadOverride.FUNCTIONAL))).
+                                //then(literal("functional_optimized").executes((cc) -> ss(cc).addScriptHost(cc.getSource(), StringArgumentType.getString(cc, "app"), null, false, false, false, null, Expression.LoadOverride.FUNCTIONAL_OPTIMIZED)))
                         )
                 );
         LiteralArgumentBuilder<CommandSourceStack> f = literal("unload").requires(Vanilla::ServerPlayer_canScriptACE).
@@ -366,25 +366,24 @@ public class ScriptCommand
                         }));
 
         LiteralArgumentBuilder<CommandSourceStack> x = literal("explain").requires(Vanilla::ServerPlayer_canScriptACE).
-                then(argument("style", StringArgumentType.word()).suggests((cc, bb) -> suggest(List.of("raw", "clean", "functional", "canonical", "optimized"), bb)).
-                        executes((cc) -> explain(
-                                cc,
-                                null, null, StringArgumentType.getString(cc, "style"))).
-                        then(literal("run").then(argument("expr", StringArgumentType.greedyString()).suggests(ScriptCommand::suggestCode).
-                                executes((cc) -> explain(
-                                        cc,
-                                        StringArgumentType.getString(cc, "expr"), null, StringArgumentType.getString(cc, "style"))))).
+                        executes((cc) -> explain(cc, null, null, null)).
+                        then(literal("expression").
+                                then(argument("style", StringArgumentType.word()).suggests((cc, bb) -> suggest(List.of("raw", "clean", "functional", "canonical", "optimized", "functional_optimized"), bb)).
+                                        then(argument("expr", StringArgumentType.greedyString()).suggests(ScriptCommand::suggestCode).
+                                                executes((cc) -> explain(cc, StringArgumentType.getString(cc, "expr"), null, StringArgumentType.getString(cc, "style")))
+                                        )
+                                )
+                        ).
                         then(literal("invoke").
                                 then(argument("call", StringArgumentType.word()).suggests((cc, bb) -> suggest(suggestFunctionCalls(cc), bb)).
-                                        executes((cc) -> explain(
-                                                cc,
-                                                null,
-                                                StringArgumentType.getString(cc, "call"),
-                                                StringArgumentType.getString(cc, "style")
-                                        ))
+                                        executes((cc) -> explain(cc, null, StringArgumentType.getString(cc, "call"), null))
                                 )
-                        )
-                );
+                        ).
+                        then(literal("source").
+                                then(argument("style", StringArgumentType.word()).suggests((cc, bb) -> suggest(List.of("raw", "clean", "functional", "canonical", "optimized", "functional_optimized"), bb)).
+                                        executes((cc) -> explain(cc, null, null, StringArgumentType.getString(cc, "style")))
+                                )
+                        );
 
         dispatcher.register(literal("script").
                 requires(Vanilla::ServerPlayer_canScriptGeneral).
@@ -559,17 +558,19 @@ public class ScriptCommand
         CarpetScriptHost host = getHost(context);
         return handleCall(source, host, () -> {
             CarpetExpression ex = new CarpetExpression(host.main, expr, source, new BlockPos(0, 0, 0));
-            return ex.scriptRunCommand(host, BlockPos.containing(source.getPosition()));
+            return ex.scriptRunCommand(host, BlockPos.containing(source.getPosition())).getLeft();
         });
     }
 
-    private static int explain(CommandContext<CommandSourceStack> context, @Nullable String expr, @Nullable String method, String style) throws CommandSyntaxException
+    private static int explain(CommandContext<CommandSourceStack> context, @Nullable String expr, @Nullable String method, @Nullable String style) throws CommandSyntaxException
     {
         CommandSourceStack source = context.getSource();
         CarpetScriptHost host = getHost(context);
         return handleCall(source, host, () -> {
-            CarpetExpression ex = new CarpetExpression(host.main, expr == null ? ( host.main == null ? "" : host.main.code() ) : expr, source, BlockPos.ZERO);
-            List<Token> results = ex.explain(host, method, style, BlockPos.containing(source.getPosition()));
+            // expression is irrelevant as we pass approprirate strings for parsing directly
+            // we only really need access to operators and functions?
+            CarpetExpression ex = new CarpetExpression(host.main, "", source, BlockPos.ZERO);
+            List<Token> results = ex.explain(host, expr != null ? expr : (style != null? (host.main != null ? host.main.code() : "") :null), method, style, BlockPos.containing(source.getPosition()));
             prettyPrintTokens(source, results);
             return NumericValue.of(results.size());
         });
@@ -584,6 +585,7 @@ public class ScriptCommand
         }
         List<Integer> lines = new ArrayList<>(indents.keySet());
         Collections.sort(lines);
+        int lineMax = String.valueOf(lines.getLast()).length();
         Deque<Integer> indentsStack = new ArrayDeque<>(lines);
 
         List<Component> elements = new ArrayList<>();
@@ -609,7 +611,8 @@ public class ScriptCommand
                 }
 
                 int lastPos = indents.getOrDefault(token.lineno, 0);
-                elements.add(Carpet.Messenger_compose("w " + " ".repeat(lastPos)));
+                elements.add(Carpet.Messenger_compose("fi " + " ".repeat(lineMax - String.valueOf(token.lineno).length()) + token.lineno));
+                elements.add(Carpet.Messenger_compose("g | " + " ".repeat(lastPos)));
             }
             elements.add(tokenToComponent(token));
         }
@@ -625,7 +628,7 @@ public class ScriptCommand
         }
         if (token.comment.isEmpty())
         {
-            return Carpet.Messenger_compose(styleForToken(token) + " " + surface);
+            return Carpet.Messenger_compose(styleForToken(token) + " " + surface);//, "^gi " + String.format("%s: line %s, pos %s", token.surface, token.lineno, token.linepos));
         }
         return Carpet.Messenger_compose(styleForToken(token)+"u " + surface, "^gi " + token.comment);
     }
@@ -642,7 +645,7 @@ public class ScriptCommand
             case Token.TokenType.VARIABLE -> "y";
             case Token.TokenType.STRINGPARAM -> "c";
             default -> // marker
-                    "f";
+                    "r";
         };
     }
 

src/main/java/carpet/script/ScriptHost.java

@@ -37,6 +37,9 @@ public abstract class ScriptHost
     private final ScriptServer scriptServer;
     protected boolean inTermination = false;
     public boolean strict;
+    public Expression.LoadOverride loadOverrides;
+    @Nullable
+    public Expression.ExpressionNode root;
 
     private final Set<String> deprecations = new HashSet<>();
 
@@ -143,13 +146,14 @@ public abstract class ScriptHost
 
     public ErrorSnooper errorSnooper = null;
 
-    protected ScriptHost(@Nullable Module code, ScriptServer scriptServer, boolean perUser, ScriptHost parent)
+    protected ScriptHost(@Nullable Module code, ScriptServer scriptServer, boolean perUser, ScriptHost parent, Expression.LoadOverride loadOverrides)
     {
         this.parent = parent;
         this.main = code;
         this.perUser = perUser;
         this.user = null;
         this.strict = false;
+        this.loadOverrides = loadOverrides;
         this.scriptServer = scriptServer;
         ModuleData moduleData = new ModuleData(code);
         initializeModuleGlobals(moduleData);
@@ -231,6 +235,7 @@ public abstract class ScriptHost
 
     protected abstract void runModuleCode(Context c, Module module); // this should be shell out in the executor
 
+    @Nullable
     public FunctionValue getFunction(String name)
     {
         return getFunction(main, name);
@@ -253,6 +258,7 @@ public abstract class ScriptHost
         return ret;
     }
 
+    @Nullable
     private FunctionValue getFunction(Module module, String name)
     {
         ModuleData local = getModuleData(module);
@@ -286,6 +292,7 @@ public abstract class ScriptHost
         return target.globalFunctions.get(name);
     }
 
+    @Nullable
     private ModuleData findModuleDataFromFunctionImports(String name, ModuleData source, int ttl)
     {
         if (ttl > 64)
@@ -460,6 +467,7 @@ public abstract class ScriptHost
         this.moduleData.forEach((key, value) -> host.moduleData.put(key, new ModuleData(key, value)));
         // fixing imports
         host.moduleData.forEach((module, data) -> data.setImportsBasedOn(host, this.moduleData.get(data.parent)));
+        host.root = this.root;
     }
 
     public synchronized void handleExpressionException(String msg, ExpressionException exc)

src/main/java/carpet/script/Token.java

@@ -35,6 +35,10 @@ public class Token implements Comparable<Token> {
     public int ordinal = 0;
     public int linepos;
     public int lineno;
+
+    @Nullable
+    public Expression.ExpressionNode node = null;
+
     public static final Token NONE = new Token();
 
     public Token morphedInto(TokenType newType, String newSurface) {
@@ -44,6 +48,7 @@ public class Token implements Comparable<Token> {
         created.pos = pos;
         created.linepos = linepos;
         created.lineno = lineno;
+        created.node = node;
         created.ordinal = ordinal + 1;
         return created;
     }

src/main/java/carpet/script/Tokenizer.java

@@ -388,14 +388,8 @@ public class Tokenizer
             else
             {
                 token.type = Token.TokenType.OPERATOR;
-                if (token.surface.equals(";"))
-                {
-                    token.disguiseAs(token.surface + " ", null);
-                }
-                else
-                {
-                    token.disguiseAs(" " + token.surface + " ", null);
-                }
+                token.disguiseAs(token.surface.equals(";") ? (token.surface + " ") : (" " + token.surface + " "), null);
+
             }
         }
         if (expression != null && context != null && previousToken != null &&

src/main/java/carpet/script/utils/AppStoreManager.java

@@ -2,6 +2,7 @@ package carpet.script.utils;
 
 import carpet.script.CarpetScriptHost;
 import carpet.script.CarpetScriptServer;
+import carpet.script.Expression;
 import carpet.script.exception.InternalExpressionException;
 import carpet.script.external.Carpet;
 import carpet.script.external.Vanilla;
@@ -283,8 +284,7 @@ public class AppStoreManager
         {
             return 0;
         }
-        boolean success = Vanilla.MinecraftServer_getScriptServer(source.getServer()).addScriptHost(source, nodeInfo.name().replaceFirst("\\.sc$", ""), null, true, false, false, nodeInfo.source());
-        return success ? 1 : 0;
+        return Vanilla.MinecraftServer_getScriptServer(source.getServer()).addScriptHost(source, nodeInfo.name().replaceFirst("\\.sc$", ""), null, true, false, false, nodeInfo.source(), Expression.LoadOverride.DEFAULT);
     }
 
     /**