From 14be0aef52d2e147398686a9674d3591fa31f461 Mon Sep 17 00:00:00 2001 From: gnembon <41132274+gnembon@users.noreply.github.com> Date: Sun, 16 Feb 2025 18:57:19 +0100 Subject: [PATCH] extract commonly used class from the Tokenizer --- .../java/carpet/script/CarpetScriptHost.java | 2 +- src/main/java/carpet/script/Expression.java | 140 +++++++++--------- src/main/java/carpet/script/Fluff.java | 10 +- .../java/carpet/script/ScriptCommand.java | 4 +- src/main/java/carpet/script/ScriptHost.java | 2 +- src/main/java/carpet/script/Token.java | 69 +++++++++ src/main/java/carpet/script/Tokenizer.java | 81 +--------- .../script/exception/ExpressionException.java | 16 +- .../InternalExpressionException.java | 4 +- .../exception/ProcessedThrowStatement.java | 2 +- .../script/exception/ThrowStatement.java | 2 +- .../carpet/script/language/Functions.java | 4 +- .../carpet/script/value/FunctionValue.java | 14 +- .../java/carpet/script/value/ThreadValue.java | 6 +- 14 files changed, 172 insertions(+), 184 deletions(-) create mode 100644 src/main/java/carpet/script/Token.java diff --git a/src/main/java/carpet/script/CarpetScriptHost.java b/src/main/java/carpet/script/CarpetScriptHost.java index 7a8d8412..a4c05d6c 100644 --- a/src/main/java/carpet/script/CarpetScriptHost.java +++ b/src/main/java/carpet/script/CarpetScriptHost.java @@ -754,7 +754,7 @@ public class CarpetScriptHost extends ScriptHost } } String sign = ""; - for (Tokenizer.Token tok : Tokenizer.simplepass(arg)) + for (Token tok : Tokenizer.simplepass(arg)) { switch (tok.type) { diff --git a/src/main/java/carpet/script/Expression.java b/src/main/java/carpet/script/Expression.java index a76fa015..90f7f0bc 100644 --- a/src/main/java/carpet/script/Expression.java +++ b/src/main/java/carpet/script/Expression.java @@ -130,7 +130,7 @@ public class Expression return constants.get(surface); } - public List getExpressionSnippet(Tokenizer.Token token) + public List getExpressionSnippet(Token token) { String code = this.getCodeString(); List output = new ArrayList<>(getExpressionSnippetLeftContext(token, code, 1)); @@ -140,7 +140,7 @@ public class Expression return output; } - private static List getExpressionSnippetLeftContext(Tokenizer.Token token, String expr, int contextsize) + private static List getExpressionSnippetLeftContext(Token token, String expr, int contextsize) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); @@ -156,7 +156,7 @@ public class Expression return output; } - private static List getExpressionSnippetContext(Tokenizer.Token token, String expr) + private static List getExpressionSnippetContext(Token token, String expr) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); @@ -173,7 +173,7 @@ public class Expression return output; } - private static List getExpressionSnippetRightContext(Tokenizer.Token token, String expr, int contextsize) + private static List getExpressionSnippetRightContext(Token token, String expr, int contextsize) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); @@ -213,7 +213,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v, LazyValue v2) + public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v, LazyValue v2) { try { @@ -229,7 +229,7 @@ public class Expression public void addLazyBinaryOperatorWithDelegation(String surface, int precedence, boolean leftAssoc, boolean pure, - SexFunction lazyfun) + SexFunction lazyfun) { operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc) { @@ -246,7 +246,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) + public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2) { try { @@ -266,7 +266,7 @@ public class Expression } public void addLazyFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive, - QuinnFunction, LazyValue> lazyfun) + QuinnFunction, LazyValue> lazyfun) { functions.put(name, new AbstractLazyFunction(numpar, name) { @@ -283,7 +283,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List lv) + public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List lv) { ILazyFunction.checkInterrupts(); try @@ -299,7 +299,7 @@ public class Expression } public void addFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive, - QuinnFunction, Value> fun) + QuinnFunction, Value> fun) { functions.put(name, new AbstractLazyFunction(numpar, name) { @@ -316,7 +316,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List lv) + public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List lv) { try { @@ -356,7 +356,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2) + public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2) { ILazyFunction.checkInterrupts(); try @@ -389,7 +389,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2) + public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2) { try { @@ -404,7 +404,7 @@ public class Expression }); } - public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Tokenizer.Token token) + public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Token token) { if (exc instanceof ExitStatement) { @@ -568,7 +568,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List lazyParams) { ILazyFunction.checkInterrupts(); if (numParams >= 0 && lazyParams.size() != numParams) @@ -606,7 +606,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List lazyParams) { ILazyFunction.checkInterrupts(); try @@ -644,7 +644,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List lazyParams) { ILazyFunction.checkInterrupts(); try @@ -676,7 +676,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List lazyParams) { ILazyFunction.checkInterrupts(); try @@ -715,7 +715,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List lazyParams) { try { @@ -730,7 +730,7 @@ public class Expression }); } - public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Tokenizer.Token token, List arguments, String varArgs, List outers, LazyValue code) + public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Token token, List arguments, String varArgs, List outers, LazyValue code) { if (functions.containsKey(name)) { @@ -799,7 +799,7 @@ public class Expression } @Override - public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List lazyParams) { c.host.issueDeprecation(copy + "(...)"); return originalFunction.lazyEval(c, type, expr, token, lazyParams); @@ -860,18 +860,18 @@ public class Expression } - private List shuntingYard(Context c) + private List shuntingYard(Context c) { - List outputQueue = new ArrayList<>(); - Stack stack = new ObjectArrayList<>(); + List outputQueue = new ArrayList<>(); + Stack stack = new ObjectArrayList<>(); Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions); // stripping lousy but acceptable semicolons - List cleanedTokens = tokenizer.postProcess(); + List cleanedTokens = tokenizer.postProcess(); - Tokenizer.Token lastFunction = null; - Tokenizer.Token previousToken = null; - for (Tokenizer.Token token : cleanedTokens) + Token lastFunction = null; + Token previousToken = null; + for (Token token : cleanedTokens) { switch (token.type) { @@ -880,9 +880,9 @@ public class Expression //break; case LITERAL, HEX_LITERAL: if (previousToken != null && ( - previousToken.type == Tokenizer.Token.TokenType.LITERAL || - previousToken.type == Tokenizer.Token.TokenType.HEX_LITERAL || - previousToken.type == Tokenizer.Token.TokenType.STRINGPARAM)) + previousToken.type == Token.TokenType.LITERAL || + previousToken.type == Token.TokenType.HEX_LITERAL || + previousToken.type == Token.TokenType.STRINGPARAM)) { throw new ExpressionException(c, this, token, "Missing operator"); } @@ -896,11 +896,11 @@ public class Expression lastFunction = token; break; case COMMA: - if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) + if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR) { throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator "); } - while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN) + while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN) { outputQueue.add(stack.pop()); } @@ -919,7 +919,7 @@ public class Expression case OPERATOR: { if (previousToken != null - && (previousToken.type == Tokenizer.Token.TokenType.COMMA || previousToken.type == Tokenizer.Token.TokenType.OPEN_PAREN)) + && (previousToken.type == Token.TokenType.COMMA || previousToken.type == Token.TokenType.OPEN_PAREN)) { throw new ExpressionException(c, this, token, "Missing parameter(s) for operator '" + token + "'"); } @@ -935,8 +935,8 @@ public class Expression } case UNARY_OPERATOR: { - if (previousToken != null && previousToken.type != Tokenizer.Token.TokenType.OPERATOR - && previousToken.type != Tokenizer.Token.TokenType.COMMA && previousToken.type != Tokenizer.Token.TokenType.OPEN_PAREN) + if (previousToken != null && previousToken.type != Token.TokenType.OPERATOR + && previousToken.type != Token.TokenType.COMMA && previousToken.type != Token.TokenType.OPEN_PAREN) { throw new ExpressionException(c, this, token, "Invalid position for unary operator " + token); } @@ -952,18 +952,18 @@ public class Expression } case OPEN_PAREN: // removed implicit multiplication in this missing code block - if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.FUNCTION) + if (previousToken != null && previousToken.type == Token.TokenType.FUNCTION) { outputQueue.add(token); } stack.push(token); break; case CLOSE_PAREN: - if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) + if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR) { throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator " + previousToken); } - while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN) + while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN) { outputQueue.add(stack.pop()); } @@ -972,7 +972,7 @@ public class Expression throw new ExpressionException(c, this, "Mismatched parentheses"); } stack.pop(); - if (!stack.isEmpty() && stack.top().type == Tokenizer.Token.TokenType.FUNCTION) + if (!stack.isEmpty() && stack.top().type == Token.TokenType.FUNCTION) { outputQueue.add(stack.pop()); } @@ -986,7 +986,7 @@ public class Expression } break; } - if (token.type != Tokenizer.Token.TokenType.MARKER) + if (token.type != Token.TokenType.MARKER) { previousToken = token; } @@ -994,8 +994,8 @@ public class Expression while (!stack.isEmpty()) { - Tokenizer.Token element = stack.pop(); - if (element.type == Tokenizer.Token.TokenType.OPEN_PAREN || element.type == Tokenizer.Token.TokenType.CLOSE_PAREN) + Token element = stack.pop(); + if (element.type == Token.TokenType.OPEN_PAREN || element.type == Token.TokenType.CLOSE_PAREN) { throw new ExpressionException(c, this, element, "Mismatched parentheses"); } @@ -1004,12 +1004,12 @@ public class Expression return outputQueue; } - private void shuntOperators(List outputQueue, Stack stack, ILazyOperator o1) + private void shuntOperators(List outputQueue, Stack stack, ILazyOperator o1) { - Tokenizer.Token nextToken = stack.isEmpty() ? null : stack.top(); + Token nextToken = stack.isEmpty() ? null : stack.top(); while (nextToken != null - && (nextToken.type == Tokenizer.Token.TokenType.OPERATOR - || nextToken.type == Tokenizer.Token.TokenType.UNARY_OPERATOR) + && (nextToken.type == Token.TokenType.OPERATOR + || nextToken.type == Token.TokenType.UNARY_OPERATOR) && ((o1.isLeftAssoc() && o1.getPrecedence() <= operators.get(nextToken.surface).getPrecedence()) || (o1.getPrecedence() < operators.get(nextToken.surface).getPrecedence()))) { @@ -1059,15 +1059,15 @@ public class Expression { public LazyValue op; public List args; - public Tokenizer.Token token; - public List range; + public Token token; + public List range; /** * The Value representation of the left parenthesis, used for parsing * varying numbers of function parameters. */ - public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Tokenizer.Token.NONE); + public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Token.NONE); - public ExpressionNode(LazyValue op, List args, Tokenizer.Token token) + public ExpressionNode(LazyValue op, List args, Token token) { this.op = op; this.args = args; @@ -1076,17 +1076,17 @@ public class Expression range.add(token); } - public static ExpressionNode ofConstant(Value val, Tokenizer.Token token) + public static ExpressionNode ofConstant(Value val, Token token) { return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token); } } - private ExpressionNode RPNToParseTree(List tokens, Context context) + private ExpressionNode RPNToParseTree(List tokens, Context context) { Stack nodeStack = new ObjectArrayList<>(); - for (Tokenizer.Token token : tokens) + for (Token token : tokens) { switch (token.type) { case UNARY_OPERATOR -> { @@ -1104,7 +1104,7 @@ public class Expression Value constant = getConstantFor(token.surface); if (constant != null) { - token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); + token.morph(Token.TokenType.CONSTANT, token.surface); nodeStack.push(new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token)); } else @@ -1135,8 +1135,8 @@ public class Expression } if (!isKnown) { - p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Tokenizer.Token.TokenType.STRINGPARAM, token.surface))); - token.morph(Tokenizer.Token.TokenType.FUNCTION, "call"); + p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Token.TokenType.STRINGPARAM, token.surface))); + token.morph(Token.TokenType.FUNCTION, "call"); } Collections.reverse(p); if (nodeStack.top() == ExpressionNode.PARAMS_START) @@ -1160,11 +1160,11 @@ public class Expression { throw new ExpressionException(context, this, token, "Not a number"); } - token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); + token.morph(Token.TokenType.CONSTANT, token.surface); nodeStack.push(ExpressionNode.ofConstant(number, token)); } case STRINGPARAM -> { - token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); + token.morph(Token.TokenType.CONSTANT, token.surface); nodeStack.push(ExpressionNode.ofConstant(new StringValue(token.surface), token)); } case HEX_LITERAL -> { @@ -1177,7 +1177,7 @@ public class Expression { throw new ExpressionException(context, this, token, "Not a number"); } - token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); + token.morph(Token.TokenType.CONSTANT, token.surface); nodeStack.push(ExpressionNode.ofConstant(hexNumber, token)); } default -> throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'"); @@ -1188,7 +1188,7 @@ public class Expression private LazyValue getAST(Context context) { - List rpn = shuntingYard(context); + List rpn = shuntingYard(context); validate(context, rpn); ExpressionNode root = RPNToParseTree(rpn, context); if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server)) @@ -1272,7 +1272,7 @@ public class Expression { // ctx is just to report errors, not values evaluation boolean optimized = false; - Tokenizer.Token.TokenType token = node.token.type; + Token.TokenType token = node.token.type; if (!token.isFunctional()) { return false; @@ -1284,7 +1284,7 @@ public class Expression } // function or operator String symbol = node.token.surface; - Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol); + Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol); Context.Type requestedType = operation.staticType(expectedType); for (ExpressionNode arg : node.args) { @@ -1326,7 +1326,7 @@ public class Expression else { returnNode.op = LazyValue.ofConstant(Value.NULL); - returnNode.token.morph(Tokenizer.Token.TokenType.CONSTANT, ""); + returnNode.token.morph(Token.TokenType.CONSTANT, ""); returnNode.args = Collections.emptyList(); if (scriptsDebugging) { @@ -1370,7 +1370,7 @@ public class Expression { CarpetScriptServer.LOG.info(" - " + symbol + "(" + node.args.size() + ") => " + function + "(" + newargs.size() + ") at line " + (node.token.lineno + 1) + ", node depth " + indent); } - node.token.morph(Tokenizer.Token.TokenType.FUNCTION, function); + node.token.morph(Token.TokenType.FUNCTION, function); node.args = newargs; } } @@ -1382,7 +1382,7 @@ public class Expression { // ctx is just to report errors, not values evaluation boolean optimized = false; - Tokenizer.Token.TokenType token = node.token.type; + Token.TokenType token = node.token.type; if (!token.isFunctional()) { return false; @@ -1396,7 +1396,7 @@ public class Expression } // function or operator - Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol); + Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol); Context.Type requestedType = operation.staticType(expectedType); for (ExpressionNode arg : node.args) { @@ -1491,7 +1491,7 @@ public class Expression Value ret = ((LazyValue.ContextFreeLazyValue) node.op).evalType(expectedType); return (c, t) -> ret; } - Tokenizer.Token token = node.token; + Token token = node.token; switch (token.type) { case UNARY_OPERATOR: @@ -1526,7 +1526,7 @@ public class Expression } } - private void validate(Context c, List rpn) + private void validate(Context c, List rpn) { /*- * Thanks to Norman Ramsey: @@ -1541,7 +1541,7 @@ public class Expression // push the 'global' scope stack.push(0); - for (Tokenizer.Token token : rpn) + for (Token token : rpn) { switch (token.type) { diff --git a/src/main/java/carpet/script/Fluff.java b/src/main/java/carpet/script/Fluff.java index 0ab2e11d..04286062 100644 --- a/src/main/java/carpet/script/Fluff.java +++ b/src/main/java/carpet/script/Fluff.java @@ -70,7 +70,7 @@ public abstract class Fluff boolean numParamsVaries(); - LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List lazyParams); + LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List lazyParams); static void checkInterrupts() { @@ -93,7 +93,7 @@ public abstract class Fluff boolean isLeftAssoc(); - LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2); + LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2); } public interface IOperator extends ILazyOperator @@ -187,7 +187,7 @@ public abstract class Fluff } @Override - public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, List lazyParams) { return new LazyValue() @@ -273,7 +273,7 @@ public abstract class Fluff } @Override - public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) + public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2) { return (c, typeIgnored) -> { try @@ -308,7 +308,7 @@ public abstract class Fluff } @Override - public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) + public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2) { if (v2 != null) { diff --git a/src/main/java/carpet/script/ScriptCommand.java b/src/main/java/carpet/script/ScriptCommand.java index 0a0b2188..18b5c282 100644 --- a/src/main/java/carpet/script/ScriptCommand.java +++ b/src/main/java/carpet/script/ScriptCommand.java @@ -37,9 +37,7 @@ import net.minecraft.commands.arguments.blocks.BlockStateArgument; import net.minecraft.commands.arguments.coordinates.BlockPosArgument; import net.minecraft.core.BlockPos; import net.minecraft.server.level.ServerLevel; -import net.minecraft.world.Clearable; import net.minecraft.world.level.block.Block; -import net.minecraft.world.level.block.entity.BlockEntity; import net.minecraft.world.level.block.state.pattern.BlockInWorld; import net.minecraft.world.level.levelgen.structure.BoundingBox; @@ -438,7 +436,7 @@ public class ScriptCommand return; } Expression expr = fun.getExpression(); - Tokenizer.Token tok = fun.getToken(); + Token tok = fun.getToken(); List snippet = expr.getExpressionSnippet(tok); Carpet.Messenger_message(source, "wb " + fun.fullName(), "t defined at: line " + (tok.lineno + 1) + " pos " + (tok.linepos + 1)); for (String snippetLine : snippet) diff --git a/src/main/java/carpet/script/ScriptHost.java b/src/main/java/carpet/script/ScriptHost.java index 29cd5233..d067d2ab 100644 --- a/src/main/java/carpet/script/ScriptHost.java +++ b/src/main/java/carpet/script/ScriptHost.java @@ -138,7 +138,7 @@ public abstract class ScriptHost @FunctionalInterface public interface ErrorSnooper { - List apply(Expression expression, Tokenizer.Token token, Context context, String message); + List apply(Expression expression, Token token, Context context, String message); } public ErrorSnooper errorSnooper = null; diff --git a/src/main/java/carpet/script/Token.java b/src/main/java/carpet/script/Token.java new file mode 100644 index 00000000..1340f4b3 --- /dev/null +++ b/src/main/java/carpet/script/Token.java @@ -0,0 +1,69 @@ +package carpet.script; + +public class Token { + enum TokenType { + FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false), + VARIABLE(false, false), CONSTANT(false, true), + LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true), + OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true); + + final boolean functional; + final boolean constant; + + TokenType(boolean functional, boolean constant) { + this.functional = functional; + this.constant = constant; + } + + public boolean isFunctional() { + return functional; + } + + public boolean isConstant() { + return constant; + } + } + + public String surface = ""; + public TokenType type; + public int pos; + public int linepos; + public int lineno; + public static final Token NONE = new Token(); + + public Token morphedInto(TokenType newType, String newSurface) { + Token created = new Token(); + created.surface = newSurface; + created.type = newType; + created.pos = pos; + created.linepos = linepos; + created.lineno = lineno; + return created; + } + + public void morph(TokenType type, String s) { + this.type = type; + this.surface = s; + } + + public void append(char c) { + surface += c; + } + + public void append(String s) { + surface += s; + } + + public char charAt(int pos) { + return surface.charAt(pos); + } + + public int length() { + return surface.length(); + } + + @Override + public String toString() { + return surface; + } +} diff --git a/src/main/java/carpet/script/Tokenizer.java b/src/main/java/carpet/script/Tokenizer.java index 90994f00..21917031 100644 --- a/src/main/java/carpet/script/Tokenizer.java +++ b/src/main/java/carpet/script/Tokenizer.java @@ -14,7 +14,7 @@ import java.util.stream.StreamSupport; * Expression tokenizer that allows to iterate over a {@link String} * expression token by token. Blank characters will be skipped. */ -public class Tokenizer implements Iterator +public class Tokenizer implements Iterator { /** * What character to use for decimal separators. @@ -421,83 +421,4 @@ public class Tokenizer implements Iterator throw new InternalExpressionException("remove() not supported"); } - public static class Token - { - enum TokenType - { - FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false), - VARIABLE(false, false), CONSTANT(false, true), - LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true), - OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true); - - final boolean functional; - final boolean constant; - - TokenType(boolean functional, boolean constant) - { - this.functional = functional; - this.constant = constant; - } - - public boolean isFunctional() - { - return functional; - } - - public boolean isConstant() - { - return constant; - } - } - - public String surface = ""; - public TokenType type; - public int pos; - public int linepos; - public int lineno; - public static final Token NONE = new Token(); - - public Token morphedInto(TokenType newType, String newSurface) - { - Token created = new Token(); - created.surface = newSurface; - created.type = newType; - created.pos = pos; - created.linepos = linepos; - created.lineno = lineno; - return created; - } - - public void morph(TokenType type, String s) - { - this.type = type; - this.surface = s; - } - - public void append(char c) - { - surface += c; - } - - public void append(String s) - { - surface += s; - } - - public char charAt(int pos) - { - return surface.charAt(pos); - } - - public int length() - { - return surface.length(); - } - - @Override - public String toString() - { - return surface; - } - } } diff --git a/src/main/java/carpet/script/exception/ExpressionException.java b/src/main/java/carpet/script/exception/ExpressionException.java index d0ab1650..a2cdd1f0 100644 --- a/src/main/java/carpet/script/exception/ExpressionException.java +++ b/src/main/java/carpet/script/exception/ExpressionException.java @@ -2,7 +2,7 @@ package carpet.script.exception; import carpet.script.Context; import carpet.script.Expression; -import carpet.script.Tokenizer; +import carpet.script.Token; import carpet.script.external.Carpet; import carpet.script.value.FunctionValue; @@ -16,7 +16,7 @@ import java.util.function.Supplier; public class ExpressionException extends StacklessRuntimeException implements ResolvedException { public final Context context; - public final Tokenizer.Token token; + public final Token token; public final List stack = new ArrayList<>(); private final Supplier lazyMessage; private String cachedMessage = null; @@ -28,15 +28,15 @@ public class ExpressionException extends StacklessRuntimeException implements Re public ExpressionException(Context c, Expression e, String message) { - this(c, e, Tokenizer.Token.NONE, message); + this(c, e, Token.NONE, message); } - public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message) + public ExpressionException(Context c, Expression e, Token t, String message) { this(c, e, t, message, Collections.emptyList()); } - public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message, List stack) + public ExpressionException(Context c, Expression e, Token t, String message, List stack) { super("Error"); this.stack.addAll(stack); @@ -45,7 +45,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re context = c; } - public ExpressionException(Context c, Expression e, Tokenizer.Token t, Supplier messageSupplier, List stack) + public ExpressionException(Context c, Expression e, Token t, Supplier messageSupplier, List stack) { super("Error"); this.stack.addAll(stack); @@ -54,7 +54,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re context = c; } - private static List makeError(Expression expr, @Nullable Tokenizer.Token token, String errmessage) + private static List makeError(Expression expr, @Nullable Token token, String errmessage) { List errMsg = new ArrayList<>(); errmessage += expr.getModuleName() == null ? "" : (" in " + expr.getModuleName()); @@ -76,7 +76,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re return errMsg; } - static synchronized String makeMessage(Context c, Expression e, Tokenizer.Token t, String message) throws ExpressionException + static synchronized String makeMessage(Context c, Expression e, Token t, String message) throws ExpressionException { if (c.getErrorSnooper() != null) { diff --git a/src/main/java/carpet/script/exception/InternalExpressionException.java b/src/main/java/carpet/script/exception/InternalExpressionException.java index ef850ed1..c37acde7 100644 --- a/src/main/java/carpet/script/exception/InternalExpressionException.java +++ b/src/main/java/carpet/script/exception/InternalExpressionException.java @@ -2,7 +2,7 @@ package carpet.script.exception; import carpet.script.Context; import carpet.script.Expression; -import carpet.script.Tokenizer; +import carpet.script.Token; import carpet.script.value.FunctionValue; import java.util.ArrayList; @@ -30,7 +30,7 @@ public class InternalExpressionException extends StacklessRuntimeException * @return The new {@link ExpressionException} (or {@link ProcessedThrowStatement}), * depending on the implementation. */ - public ExpressionException promote(Context c, Expression e, Tokenizer.Token token) + public ExpressionException promote(Context c, Expression e, Token token) { return new ExpressionException(c, e, token, getMessage(), stack); } diff --git a/src/main/java/carpet/script/exception/ProcessedThrowStatement.java b/src/main/java/carpet/script/exception/ProcessedThrowStatement.java index 3768550b..7d5cc7fc 100644 --- a/src/main/java/carpet/script/exception/ProcessedThrowStatement.java +++ b/src/main/java/carpet/script/exception/ProcessedThrowStatement.java @@ -4,7 +4,7 @@ import java.util.List; import carpet.script.Context; import carpet.script.Expression; -import carpet.script.Tokenizer.Token; +import carpet.script.Token; import carpet.script.value.FunctionValue; import carpet.script.value.Value; diff --git a/src/main/java/carpet/script/exception/ThrowStatement.java b/src/main/java/carpet/script/exception/ThrowStatement.java index 40cf10b9..67abccb0 100644 --- a/src/main/java/carpet/script/exception/ThrowStatement.java +++ b/src/main/java/carpet/script/exception/ThrowStatement.java @@ -2,7 +2,7 @@ package carpet.script.exception; import carpet.script.Context; import carpet.script.Expression; -import carpet.script.Tokenizer.Token; +import carpet.script.Token; import carpet.script.value.StringValue; import carpet.script.value.Value; diff --git a/src/main/java/carpet/script/language/Functions.java b/src/main/java/carpet/script/language/Functions.java index 8bc29712..509b7270 100644 --- a/src/main/java/carpet/script/language/Functions.java +++ b/src/main/java/carpet/script/language/Functions.java @@ -4,7 +4,7 @@ import carpet.script.Context; import carpet.script.Expression; import carpet.script.Fluff; import carpet.script.LazyValue; -import carpet.script.Tokenizer; +import carpet.script.Token; import carpet.script.argument.FunctionArgument; import carpet.script.exception.InternalExpressionException; import carpet.script.exception.ReturnStatement; @@ -45,7 +45,7 @@ public class Functions expression.addCustomFunction("call", new Fluff.AbstractLazyFunction(-1, "call") { @Override - public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Tokenizer.Token tok, List lv) + public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Token tok, List lv) { if (lv.isEmpty()) { diff --git a/src/main/java/carpet/script/value/FunctionValue.java b/src/main/java/carpet/script/value/FunctionValue.java index 1cc7f449..db54f1aa 100644 --- a/src/main/java/carpet/script/value/FunctionValue.java +++ b/src/main/java/carpet/script/value/FunctionValue.java @@ -6,7 +6,7 @@ import carpet.script.Expression; import carpet.script.Fluff; import carpet.script.LazyValue; import carpet.script.Module; -import carpet.script.Tokenizer; +import carpet.script.Token; import carpet.script.exception.BreakStatement; import carpet.script.exception.ContinueStatement; import carpet.script.exception.ExpressionException; @@ -28,7 +28,7 @@ import javax.annotation.Nullable; public class FunctionValue extends Value implements Fluff.ILazyFunction { private final Expression expression; - private final Tokenizer.Token token; + private final Token token; private final String name; private final LazyValue body; private Map outerState; @@ -37,7 +37,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction private static long variantCounter = 1; private long variant; - private FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List args, String varArgs) + private FunctionValue(Expression expression, Token token, String name, LazyValue body, List args, String varArgs) { this.expression = expression; this.token = token; @@ -49,7 +49,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction variant = 0L; } - public FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List args, String varArgs, Map outerState) + public FunctionValue(Expression expression, Token token, String name, LazyValue body, List args, String varArgs, Map outerState) { this.expression = expression; this.token = token; @@ -214,13 +214,13 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction } @Override - public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List lazyParams) + public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List lazyParams) { List resolvedParams = unpackArgs(lazyParams, c); return execute(c, type, e, t, resolvedParams, null); } - public LazyValue execute(Context c, Context.Type type, Expression e, Tokenizer.Token t, List params, @Nullable ThreadValue freshNewCallingThread) + public LazyValue execute(Context c, Context.Type type, Expression e, Token t, List params, @Nullable ThreadValue freshNewCallingThread) { assertArgsOk(params, fixedArgs -> { if (fixedArgs) // wrong number of args for fixed args @@ -288,7 +288,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction return expression; } - public Tokenizer.Token getToken() + public Token getToken() { return token; } diff --git a/src/main/java/carpet/script/value/ThreadValue.java b/src/main/java/carpet/script/value/ThreadValue.java index e5665557..793a2d5f 100644 --- a/src/main/java/carpet/script/value/ThreadValue.java +++ b/src/main/java/carpet/script/value/ThreadValue.java @@ -2,7 +2,7 @@ package carpet.script.value; import carpet.script.Context; import carpet.script.Expression; -import carpet.script.Tokenizer; +import carpet.script.Token; import carpet.script.exception.ExitStatement; import carpet.script.exception.ExpressionException; import carpet.script.exception.InternalExpressionException; @@ -28,7 +28,7 @@ public class ThreadValue extends LazyListValue private final AtomicReference coLock = new AtomicReference<>(Value.EOL); public final boolean isCoroutine; - public ThreadValue(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List args) + public ThreadValue(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List args) { this.id = sequence++; this.isCoroutine = ctx.host.canSynchronouslyExecute(); @@ -37,7 +37,7 @@ public class ThreadValue extends LazyListValue Thread.yield(); } - public CompletableFuture getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List args) + public CompletableFuture getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List args) { ExecutorService executor = ctx.host.getExecutor(pool); ThreadValue callingThread = isCoroutine ? this : null;