improve explain bits to be usable

src/main/java/carpet/script/CarpetExpression.java

@@ -154,7 +154,7 @@ public class CarpetExpression
         }
     }
 
-    public List<Token> explain(ScriptHost host, BlockPos pos)
+    public List<Token> explain(ScriptHost host, @Nullable String method, @Nullable String style, BlockPos pos)
     {
         CarpetScriptServer scriptServer = (CarpetScriptServer) host.scriptServer();
         try
@@ -174,7 +174,7 @@ public class CarpetExpression
                 Value playerValue = new EntityValue(e).bindTo("p");
                 context.with("p", (cc, tt) -> playerValue);
             }
-            return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.explain(context));
+            return scriptServer.events.handleEvents.getWhileDisabled(() -> this.expr.explain(context, method, style));
         }
         catch (ExpressionException e)
         {

src/main/java/carpet/script/Expression.java

@@ -1535,22 +1535,31 @@ public class Expression
         }
     }
 
-    public List<Token> explain(Context context)
+    public List<Token> explain(Context context, @Nullable String method, @Nullable String style)
     {
         //todo convert to explain
         Tokenizer tokenizer = new Tokenizer(context, this, expression, true, true);
         // stripping lousy but acceptable semicolons
         List<Token> input = tokenizer.parseTokens();
+        if (style.equalsIgnoreCase("raw")) {
+            return input;
+        }
         List<Token> cleanedTokens = Tokenizer.postProcess(input);
+        if (style.equalsIgnoreCase("clean")) {
+            return cleanedTokens;
+        }
 
         List<Token> rpn = shuntingYard(context, cleanedTokens);
         validate(context, rpn);
         ExpressionNode root = RPNToParseTree(rpn, context);
         List<Token> parsedTokens = root.tokensRecursive(this);
+        if (style.equalsIgnoreCase("canonical")) {
+            return parsedTokens;
+        }
 
         Context optimizeOnlyContext = new Context.ContextForErrorReporting(context);
         // pure functional
-        optimizeTree(root, optimizeOnlyContext, null, true);
+        optimizeTree(root, optimizeOnlyContext, null,  style.equalsIgnoreCase("functional"));
         List<Token> compileTimeOptimized = root.tokensRecursive(this);
 
         return compileTimeOptimized;

src/main/java/carpet/script/ScriptCommand.java

@@ -364,15 +364,27 @@ public class ScriptCommand
                             boolean success = ss(cc).uninstallApp(cc.getSource(), StringArgumentType.getString(cc, "app"));
                             return success ? 1 : 0;
                         }));
-        LiteralArgumentBuilder<CommandSourceStack> x = literal("explain").
-                requires(Vanilla::ServerPlayer_canScriptACE).
-                executes((cc) -> explain(
-                        cc,
-                        null)).
-                then(argument("expr", StringArgumentType.greedyString()).suggests(ScriptCommand::suggestCode).
+
+        LiteralArgumentBuilder<CommandSourceStack> x = literal("explain").requires(Vanilla::ServerPlayer_canScriptACE).
+                then(argument("style", StringArgumentType.word()).suggests((cc, bb) -> suggest(List.of("raw", "clean", "functional", "canonical", "optimized"), bb)).
                         executes((cc) -> explain(
                                 cc,
-                                StringArgumentType.getString(cc, "expr"))));
+                                null, null, StringArgumentType.getString(cc, "style"))).
+                        then(literal("run").then(argument("expr", StringArgumentType.greedyString()).suggests(ScriptCommand::suggestCode).
+                                executes((cc) -> explain(
+                                        cc,
+                                        StringArgumentType.getString(cc, "expr"), null, StringArgumentType.getString(cc, "style"))))).
+                        then(literal("invoke").
+                                then(argument("call", StringArgumentType.word()).suggests((cc, bb) -> suggest(suggestFunctionCalls(cc), bb)).
+                                        executes((cc) -> explain(
+                                                cc,
+                                                null,
+                                                StringArgumentType.getString(cc, "call"),
+                                                StringArgumentType.getString(cc, "style")
+                                        ))
+                                )
+                        )
+                );
 
         dispatcher.register(literal("script").
                 requires(Vanilla::ServerPlayer_canScriptGeneral).
@@ -551,13 +563,13 @@ public class ScriptCommand
         });
     }
 
-    private static int explain(CommandContext<CommandSourceStack> context, @Nullable String expr) throws CommandSyntaxException
+    private static int explain(CommandContext<CommandSourceStack> context, @Nullable String expr, @Nullable String method, String style) throws CommandSyntaxException
     {
         CommandSourceStack source = context.getSource();
         CarpetScriptHost host = getHost(context);
         return handleCall(source, host, () -> {
-            CarpetExpression ex = new CarpetExpression(host.main, expr == null ? ( host.main == null ? "" : host.main.code() ) : expr, source, new BlockPos(0, 0, 0));
-            List<Token> results = ex.explain(host, BlockPos.containing(source.getPosition()));
+            CarpetExpression ex = new CarpetExpression(host.main, expr == null ? ( host.main == null ? "" : host.main.code() ) : expr, source, BlockPos.ZERO);
+            List<Token> results = ex.explain(host, method, style, BlockPos.containing(source.getPosition()));
             prettyPrintTokens(source, results);
             return NumericValue.of(results.size());
         });

src/main/java/carpet/script/Tokenizer.java

@@ -255,6 +255,7 @@ public class Tokenizer
             }
             pos++;
             linepos++;
+            token.disguiseAs("'"+token.surface+"'", null);
 
         }
         else if (Character.isLetter(ch) || "_".indexOf(ch) >= 0)
@@ -298,6 +299,7 @@ public class Tokenizer
             else if (ch == ',')
             {
                 token.type = Token.TokenType.COMMA;
+                token.disguiseAs(", ", null);
             }
             else
             {
@@ -379,12 +381,21 @@ public class Tokenizer
                     || (previousToken.type == Token.TokenType.MARKER && (previousToken.surface.equals("{") || previousToken.surface.equals("[")))
             )
             {
+                token.disguiseAs(token.surface, null);
                 token.surface += "u";
                 token.type = Token.TokenType.UNARY_OPERATOR;
             }
             else
             {
                 token.type = Token.TokenType.OPERATOR;
+                if (token.surface.equals(";"))
+                {
+                    token.disguiseAs(token.surface + " ", null);
+                }
+                else
+                {
+                    token.disguiseAs(" " + token.surface + " ", null);
+                }
             }
         }
         if (expression != null && context != null && previousToken != null &&