explain outputs functional target

src/main/java/carpet/script/CarpetExpression.java

@@ -153,9 +153,8 @@ public class CarpetExpression
         }
     }
 
-    public Value explain(ScriptHost host, BlockPos pos)
+    public List<Token> explain(ScriptHost host, BlockPos pos)
     {
-        // convert to explain
         CarpetScriptServer scriptServer = (CarpetScriptServer) host.scriptServer();
         try
         {

src/main/java/carpet/script/Expression.java

@@ -1298,6 +1298,61 @@ public class Expression
         {
             return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token);
         }
+
+        public List<Token> tokensRecursive(Expression expression) {
+            List<Token> tokens = new ArrayList<>();
+            switch (token.type) {
+                case FUNCTION -> {
+                    tokens.add(token);
+                    tokens.add(token.morphedInto(Token.TokenType.OPEN_PAREN, "("));
+                    if (!args.isEmpty()) {
+                        for (ExpressionNode arg : args) {
+                            List<Token> argTokens = arg.tokensRecursive(expression);
+                            tokens.addAll(argTokens);
+                            tokens.add(argTokens.getLast().morphedInto(Token.TokenType.COMMA, ",").disguiseAs(", ", null));
+                        }
+                        tokens.removeLast();
+                    }
+                    tokens.add(tokens.getLast().morphedInto(Token.TokenType.CLOSE_PAREN, ")"));
+                }
+                case OPERATOR -> {
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0)));
+                    tokens.add(token.disguiseAs(" "+token.surface+" ", null));
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(1)));
+                }
+                case UNARY_OPERATOR -> {
+                    tokens.add(token.disguiseAs(" "+StringUtils.chop(token.surface),null));
+                    tokens.addAll(createOperatorArgumentTokens(expression, args.get(0)));
+                }
+                case VARIABLE, LITERAL, HEX_LITERAL, CONSTANT -> tokens.add(token);
+                case STRINGPARAM -> tokens.add(token.disguiseAs("'"+token.surface+"'", null));
+                // these should be parsed out already, but ....
+                //case LITERAL, STRINGPARAM, HEX_LITERAL, OPEN_PAREN, CLOSE_PAREN, MARKER -> tokens.add(token.morphedInto(token.type, token.surface+"?"));
+                // CONSTANT ??
+                default -> tokens.add(token.disguiseAs("?"+token.surface+"?", null));
+            }
+            return tokens;
+        }
+
+        public List<Token> createOperatorArgumentTokens(Expression expression, ExpressionNode operand) {
+            boolean needsBrackets = false;
+            var operators = expression.operators;
+            if (operators.containsKey(operand.token.surface) && operators.containsKey(token.surface)) {
+                if (operators.get(operand.token.surface).getPrecedence() < operators.get(token.surface).getPrecedence()) {
+                    needsBrackets = true;
+                }
+            }
+            List<Token> argumentTokens = operand.tokensRecursive(expression);
+            if (!needsBrackets) {
+                return argumentTokens;
+            }
+            List<Token> bracketed = new ArrayList<>();
+            bracketed.add(argumentTokens.getFirst().morphedInto(Token.TokenType.OPEN_PAREN, "("));
+            bracketed.get(0).swapPlace(argumentTokens.getFirst());
+            bracketed.addAll(argumentTokens);
+            bracketed.add(argumentTokens.getLast().morphedInto(Token.TokenType.CLOSE_PAREN, ")"));
+            return bracketed;
+        }
     }
 
 
@@ -1478,10 +1533,25 @@ public class Expression
         }
     }
 
-    public Value explain(Context context)
+    public List<Token> explain(Context context)
     {
         //todo convert to explain
-        return Value.NULL;
+        Tokenizer tokenizer = new Tokenizer(context, this, expression, true, true);
+        // stripping lousy but acceptable semicolons
+        List<Token> input = tokenizer.parseTokens();
+        List<Token> cleanedTokens = Tokenizer.postProcess(input);
+
+        List<Token> rpn = shuntingYard(context, cleanedTokens);
+        validate(context, rpn);
+        ExpressionNode root = RPNToParseTree(rpn, context);
+        List<Token> parsedTokens = root.tokensRecursive(this);
+
+        Context optimizeOnlyContext = new Context.ContextForErrorReporting(context);
+        // pure functional
+        optimizeTree(root, optimizeOnlyContext, null, true);
+        List<Token> compileTimeOptimized = root.tokensRecursive(this);
+
+        return compileTimeOptimized;
     }
 
     private int treeSize(ExpressionNode node)
@@ -1700,6 +1770,7 @@ public class Expression
         {
             result = ((ILazyOperator) operation).lazyEval(ctx, expectedType, this, node.token, args.get(0), args.get(1)).evalValue(null, expectedType);
         }
+        node.token.disguiseAs(null, "=>"+result.getString());
         node.op = LazyValue.ofConstant(result);
         if (logger != null)
         {

src/main/java/carpet/script/ScriptCommand.java

@@ -15,14 +15,17 @@ import com.mojang.brigadier.exceptions.CommandSyntaxException;
 import com.mojang.brigadier.suggestion.Suggestions;
 import com.mojang.brigadier.suggestion.SuggestionsBuilder;
 import net.minecraft.commands.CommandBuildContext;
+import net.minecraft.network.chat.Component;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.CompletableFuture;
@@ -41,6 +44,8 @@ import net.minecraft.world.level.block.Block;
 import net.minecraft.world.level.block.state.pattern.BlockInWorld;
 import net.minecraft.world.level.levelgen.structure.BoundingBox;
 
+import javax.annotation.Nullable;
+
 import static net.minecraft.commands.Commands.argument;
 import static net.minecraft.commands.Commands.literal;
 import static net.minecraft.commands.SharedSuggestionProvider.suggest;
@@ -359,6 +364,9 @@ public class ScriptCommand
                         }));
         LiteralArgumentBuilder<CommandSourceStack> x = literal("explain").
                 requires(Vanilla::ServerPlayer_canScriptACE).
+                executes((cc) -> explain(
+                        cc,
+                        null)).
                 then(argument("expr", StringArgumentType.greedyString()).suggests(ScriptCommand::suggestCode).
                         executes((cc) -> explain(
                                 cc,
@@ -541,19 +549,88 @@ public class ScriptCommand
         });
     }
 
-    private static int explain(CommandContext<CommandSourceStack> context, String expr) throws CommandSyntaxException
+    private static int explain(CommandContext<CommandSourceStack> context, @Nullable String expr) throws CommandSyntaxException
     {
         CommandSourceStack source = context.getSource();
         CarpetScriptHost host = getHost(context);
         return handleCall(source, host, () -> {
-            CarpetExpression ex = new CarpetExpression(host.main, expr, source, new BlockPos(0, 0, 0));
+            CarpetExpression ex = new CarpetExpression(host.main, expr == null ? ( host.main == null ? "" : host.main.code() ) : expr, source, new BlockPos(0, 0, 0));
-            ex.explain(host, BlockPos.containing(source.getPosition()));
+            List<Token> results = ex.explain(host, BlockPos.containing(source.getPosition()));
-            Carpet.Messenger_message(source, "w Expression: ", "wb " + expr);
+            prettyPrintTokens(source, results);
-            Carpet.Messenger_message(source, "w Tokens: ", "wb " + ex.getExpr().toString());
+            return NumericValue.of(results.size());
-            return NumericValue.ONE;
         });
     }
 
+    private static void prettyPrintTokens(CommandSourceStack source, List<Token> tokens) {
+        //Collections.sort(tokens);
+        // indentations per line
+        Map<Integer, Integer> indents = new HashMap<>();
+        for (Token token : tokens)
+        {
+            indents.put(token.lineno, Math.min(indents.getOrDefault(token.lineno, token.linepos), token.linepos));
+        }
+
+
+        int lastLine = 0;
+        int lastPos = 0;
+        List<Component> elements = new ArrayList<>();
+        for (Token token : tokens)
+        {
+            if (token.lineno != lastLine)
+            {
+                flushElements(source, elements);
+                lastLine = token.lineno;
+                lastPos = indents.getOrDefault(token.lineno, 0);
+                elements.add(Carpet.Messenger_compose("w " + " ".repeat(lastPos)));
+            }
+            //else if (token.linepos > lastPos)
+            //{
+            //    lastPos = token.linepos;
+            //}
+            elements.add(tokenToComponent(token));
+            lastPos += token.surface.length() + 1;
+        }
+        flushElements(source, elements);
+    }
+
+    private static Component tokenToComponent(Token token)
+    {
+        String surface = token.surface;
+        if (!token.display.isEmpty())
+        {
+            surface = token.display;
+        }
+        if (token.comment.isEmpty()) {
+            return Carpet.Messenger_compose(styleForToken(token) + " " + surface);
+        }
+        return Carpet.Messenger_compose(styleForToken(token)+"br " + surface, "^gi " + token.comment);
+    }
+
+    private static String styleForToken(Token token)
+    {
+        return switch (token.type) {
+            case Token.TokenType.LITERAL -> "l";
+            case Token.TokenType.HEX_LITERAL -> "e";
+            case Token.TokenType.CONSTANT -> "q";
+            case Token.TokenType.OPEN_PAREN, Token.TokenType.CLOSE_PAREN, Token.TokenType.COMMA,
+                 Token.TokenType.OPERATOR, Token.TokenType.UNARY_OPERATOR -> "w";
+            case Token.TokenType.FUNCTION -> "y";
+            case Token.TokenType.VARIABLE -> "y";
+            case Token.TokenType.STRINGPARAM -> "c";
+            default -> // marker
+                    "w";
+        };
+    }
+
+    private static void flushElements(CommandSourceStack source, List<Component> elements)
+    {
+        if (elements.size() > 0)
+        {
+            Carpet.Messenger_message(source, elements.toArray(new Component[0]));
+            elements.clear();
+        }
+    }
+
     private static int scriptScan(CommandContext<CommandSourceStack> context, BlockPos origin, BlockPos a, BlockPos b, String expr) throws CommandSyntaxException
     {
         CommandSourceStack source = context.getSource();

src/main/java/carpet/script/Token.java

@@ -1,6 +1,9 @@
 package carpet.script;
 
-public class Token {
+import javax.annotation.Nullable;
+
+public class Token implements Comparable<Token> {
+
     enum TokenType {
         FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
         VARIABLE(false, false), CONSTANT(false, true),
@@ -25,8 +28,11 @@ public class Token {
     }
 
     public String surface = "";
+    public String display = "";
+    public String comment = "";
     public TokenType type;
     public int pos;
+    public int ordinal = 0;
     public int linepos;
     public int lineno;
     public static final Token NONE = new Token();
@@ -38,12 +44,31 @@ public class Token {
         created.pos = pos;
         created.linepos = linepos;
         created.lineno = lineno;
+        created.ordinal = ordinal + 1;
         return created;
     }
 
+    public void swapPlace(Token other) {
+        int order = other.ordinal;
+        other.ordinal = ordinal;
+        ordinal = order;
+    }
+
     public void morph(TokenType type, String s) {
         this.type = type;
         this.surface = s;
+        this.display = "";
+        this.comment = "";
+    }
+
+    public Token disguiseAs(@Nullable String s, @Nullable String s1) {
+        if (s != null) {
+            display = s;
+        }
+        if (s1 != null) {
+            comment = s1;
+        }
+        return this;
     }
 
     public void append(char c) {
@@ -66,4 +91,18 @@ public class Token {
     public String toString() {
         return surface;
     }
+
+    @Override
+    public int compareTo(Token o) {
+        // compare by lineno, then by linepos, then by ordinal
+        if (lineno != o.lineno)
+        {
+            return lineno - o.lineno;
+        }
+        if (linepos != o.linepos)
+        {
+            return linepos - o.linepos;
+        }
+        return ordinal - o.ordinal;
+    }
 }