icybear/fabric-carpet
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

extract commonly used class from the Tokenizer

Browse Source
This commit is contained in:
gnembon committed 2025-02-16 18:57:19 +01:00
1 parent 9086db9494
commit 14be0aef52
14 files changed
+172 -184
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main/java/carpet/script/CarpetScriptHost.java
+1 -1
View File
@@ -754,7 +754,7 @@ public class CarpetScriptHost extends ScriptHost
} }
} }
String sign = ""; String sign = "";
for (Tokenizer.Token tok : Tokenizer.simplepass(arg)) for (Token tok : Tokenizer.simplepass(arg))
{ {
switch (tok.type) switch (tok.type)
{ {
src/main/java/carpet/script/Expression.java
+70 -70
View File
@@ -130,7 +130,7 @@ public class Expression
return constants.get(surface); return constants.get(surface);
} }
public List<String> getExpressionSnippet(Tokenizer.Token token) public List<String> getExpressionSnippet(Token token)
{ {
String code = this.getCodeString(); String code = this.getCodeString();
List<String> output = new ArrayList<>(getExpressionSnippetLeftContext(token, code, 1)); List<String> output = new ArrayList<>(getExpressionSnippetLeftContext(token, code, 1));
@@ -140,7 +140,7 @@ public class Expression
return output; return output;
} }
private static List<String> getExpressionSnippetLeftContext(Tokenizer.Token token, String expr, int contextsize) private static List<String> getExpressionSnippetLeftContext(Token token, String expr, int contextsize)
{ {
List<String> output = new ArrayList<>(); List<String> output = new ArrayList<>();
String[] lines = expr.split("\n"); String[] lines = expr.split("\n");
@@ -156,7 +156,7 @@ public class Expression
return output; return output;
} }
private static List<String> getExpressionSnippetContext(Tokenizer.Token token, String expr) private static List<String> getExpressionSnippetContext(Token token, String expr)
{ {
List<String> output = new ArrayList<>(); List<String> output = new ArrayList<>();
String[] lines = expr.split("\n"); String[] lines = expr.split("\n");
@@ -173,7 +173,7 @@ public class Expression
return output; return output;
} }
private static List<String> getExpressionSnippetRightContext(Tokenizer.Token token, String expr, int contextsize) private static List<String> getExpressionSnippetRightContext(Token token, String expr, int contextsize)
{ {
List<String> output = new ArrayList<>(); List<String> output = new ArrayList<>();
String[] lines = expr.split("\n"); String[] lines = expr.split("\n");
@@ -213,7 +213,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v, LazyValue v2) public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v, LazyValue v2)
{ {
try try
{ {
@@ -229,7 +229,7 @@ public class Expression
public void addLazyBinaryOperatorWithDelegation(String surface, int precedence, boolean leftAssoc, boolean pure, public void addLazyBinaryOperatorWithDelegation(String surface, int precedence, boolean leftAssoc, boolean pure,
SexFunction<Context, Context.Type, Expression, Tokenizer.Token, LazyValue, LazyValue, LazyValue> lazyfun) SexFunction<Context, Context.Type, Expression, Token, LazyValue, LazyValue, LazyValue> lazyfun)
{ {
operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc) operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc)
{ {
@@ -246,7 +246,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{ {
try try
{ {
@@ -266,7 +266,7 @@ public class Expression
} }
public void addLazyFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive, public void addLazyFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive,
QuinnFunction<Context, Context.Type, Expression, Tokenizer.Token, List<LazyValue>, LazyValue> lazyfun) QuinnFunction<Context, Context.Type, Expression, Token, List<LazyValue>, LazyValue> lazyfun)
{ {
functions.put(name, new AbstractLazyFunction(numpar, name) functions.put(name, new AbstractLazyFunction(numpar, name)
{ {
@@ -283,7 +283,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lv) public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lv)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
try try
@@ -299,7 +299,7 @@ public class Expression
} }
public void addFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive, public void addFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive,
QuinnFunction<Context, Context.Type, Expression, Tokenizer.Token, List<Value>, Value> fun) QuinnFunction<Context, Context.Type, Expression, Token, List<Value>, Value> fun)
{ {
functions.put(name, new AbstractLazyFunction(numpar, name) functions.put(name, new AbstractLazyFunction(numpar, name)
{ {
@@ -316,7 +316,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lv) public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lv)
{ {
try try
{ {
@@ -356,7 +356,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2) public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
try try
@@ -389,7 +389,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2) public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2)
{ {
try try
{ {
@@ -404,7 +404,7 @@ public class Expression
}); });
} }
public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Tokenizer.Token token) public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Token token)
{ {
if (exc instanceof ExitStatement) if (exc instanceof ExitStatement)
{ {
@@ -568,7 +568,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
if (numParams >= 0 && lazyParams.size() != numParams) if (numParams >= 0 && lazyParams.size() != numParams)
@@ -606,7 +606,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
try try
@@ -644,7 +644,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
try try
@@ -676,7 +676,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{ {
ILazyFunction.checkInterrupts(); ILazyFunction.checkInterrupts();
try try
@@ -715,7 +715,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{ {
try try
{ {
@@ -730,7 +730,7 @@ public class Expression
}); });
} }
public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Tokenizer.Token token, List<String> arguments, String varArgs, List<String> outers, LazyValue code) public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Token token, List<String> arguments, String varArgs, List<String> outers, LazyValue code)
{ {
if (functions.containsKey(name)) if (functions.containsKey(name))
{ {
@@ -799,7 +799,7 @@ public class Expression
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List<LazyValue> lazyParams)
{ {
c.host.issueDeprecation(copy + "(...)"); c.host.issueDeprecation(copy + "(...)");
return originalFunction.lazyEval(c, type, expr, token, lazyParams); return originalFunction.lazyEval(c, type, expr, token, lazyParams);
@@ -860,18 +860,18 @@ public class Expression
} }
private List<Tokenizer.Token> shuntingYard(Context c) private List<Token> shuntingYard(Context c)
{ {
List<Tokenizer.Token> outputQueue = new ArrayList<>(); List<Token> outputQueue = new ArrayList<>();
Stack<Tokenizer.Token> stack = new ObjectArrayList<>(); Stack<Token> stack = new ObjectArrayList<>();
Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions); Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions);
// stripping lousy but acceptable semicolons // stripping lousy but acceptable semicolons
List<Tokenizer.Token> cleanedTokens = tokenizer.postProcess(); List<Token> cleanedTokens = tokenizer.postProcess();
Tokenizer.Token lastFunction = null; Token lastFunction = null;
Tokenizer.Token previousToken = null; Token previousToken = null;
for (Tokenizer.Token token : cleanedTokens) for (Token token : cleanedTokens)
{ {
switch (token.type) switch (token.type)
{ {
@@ -880,9 +880,9 @@ public class Expression
//break; //break;
case LITERAL, HEX_LITERAL: case LITERAL, HEX_LITERAL:
if (previousToken != null && ( if (previousToken != null && (
previousToken.type == Tokenizer.Token.TokenType.LITERAL || previousToken.type == Token.TokenType.LITERAL ||
previousToken.type == Tokenizer.Token.TokenType.HEX_LITERAL || previousToken.type == Token.TokenType.HEX_LITERAL ||
previousToken.type == Tokenizer.Token.TokenType.STRINGPARAM)) previousToken.type == Token.TokenType.STRINGPARAM))
{ {
throw new ExpressionException(c, this, token, "Missing operator"); throw new ExpressionException(c, this, token, "Missing operator");
} }
@@ -896,11 +896,11 @@ public class Expression
lastFunction = token; lastFunction = token;
break; break;
case COMMA: case COMMA:
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR)
{ {
throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator "); throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator ");
} }
while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN) while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN)
{ {
outputQueue.add(stack.pop()); outputQueue.add(stack.pop());
} }
@@ -919,7 +919,7 @@ public class Expression
case OPERATOR: case OPERATOR:
{ {
if (previousToken != null if (previousToken != null
&& (previousToken.type == Tokenizer.Token.TokenType.COMMA || previousToken.type == Tokenizer.Token.TokenType.OPEN_PAREN)) && (previousToken.type == Token.TokenType.COMMA || previousToken.type == Token.TokenType.OPEN_PAREN))
{ {
throw new ExpressionException(c, this, token, "Missing parameter(s) for operator '" + token + "'"); throw new ExpressionException(c, this, token, "Missing parameter(s) for operator '" + token + "'");
} }
@@ -935,8 +935,8 @@ public class Expression
} }
case UNARY_OPERATOR: case UNARY_OPERATOR:
{ {
if (previousToken != null && previousToken.type != Tokenizer.Token.TokenType.OPERATOR if (previousToken != null && previousToken.type != Token.TokenType.OPERATOR
&& previousToken.type != Tokenizer.Token.TokenType.COMMA && previousToken.type != Tokenizer.Token.TokenType.OPEN_PAREN) && previousToken.type != Token.TokenType.COMMA && previousToken.type != Token.TokenType.OPEN_PAREN)
{ {
throw new ExpressionException(c, this, token, "Invalid position for unary operator " + token); throw new ExpressionException(c, this, token, "Invalid position for unary operator " + token);
} }
@@ -952,18 +952,18 @@ public class Expression
} }
case OPEN_PAREN: case OPEN_PAREN:
// removed implicit multiplication in this missing code block // removed implicit multiplication in this missing code block
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.FUNCTION) if (previousToken != null && previousToken.type == Token.TokenType.FUNCTION)
{ {
outputQueue.add(token); outputQueue.add(token);
} }
stack.push(token); stack.push(token);
break; break;
case CLOSE_PAREN: case CLOSE_PAREN:
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR)
{ {
throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator " + previousToken); throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator " + previousToken);
} }
while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN) while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN)
{ {
outputQueue.add(stack.pop()); outputQueue.add(stack.pop());
} }
@@ -972,7 +972,7 @@ public class Expression
throw new ExpressionException(c, this, "Mismatched parentheses"); throw new ExpressionException(c, this, "Mismatched parentheses");
} }
stack.pop(); stack.pop();
if (!stack.isEmpty() && stack.top().type == Tokenizer.Token.TokenType.FUNCTION) if (!stack.isEmpty() && stack.top().type == Token.TokenType.FUNCTION)
{ {
outputQueue.add(stack.pop()); outputQueue.add(stack.pop());
} }
@@ -986,7 +986,7 @@ public class Expression
} }
break; break;
} }
if (token.type != Tokenizer.Token.TokenType.MARKER) if (token.type != Token.TokenType.MARKER)
{ {
previousToken = token; previousToken = token;
} }
@@ -994,8 +994,8 @@ public class Expression
while (!stack.isEmpty()) while (!stack.isEmpty())
{ {
Tokenizer.Token element = stack.pop(); Token element = stack.pop();
if (element.type == Tokenizer.Token.TokenType.OPEN_PAREN || element.type == Tokenizer.Token.TokenType.CLOSE_PAREN) if (element.type == Token.TokenType.OPEN_PAREN || element.type == Token.TokenType.CLOSE_PAREN)
{ {
throw new ExpressionException(c, this, element, "Mismatched parentheses"); throw new ExpressionException(c, this, element, "Mismatched parentheses");
} }
@@ -1004,12 +1004,12 @@ public class Expression
return outputQueue; return outputQueue;
} }
private void shuntOperators(List<Tokenizer.Token> outputQueue, Stack<Tokenizer.Token> stack, ILazyOperator o1) private void shuntOperators(List<Token> outputQueue, Stack<Token> stack, ILazyOperator o1)
{ {
Tokenizer.Token nextToken = stack.isEmpty() ? null : stack.top(); Token nextToken = stack.isEmpty() ? null : stack.top();
while (nextToken != null while (nextToken != null
&& (nextToken.type == Tokenizer.Token.TokenType.OPERATOR && (nextToken.type == Token.TokenType.OPERATOR
|| nextToken.type == Tokenizer.Token.TokenType.UNARY_OPERATOR) || nextToken.type == Token.TokenType.UNARY_OPERATOR)
&& ((o1.isLeftAssoc() && o1.getPrecedence() <= operators.get(nextToken.surface).getPrecedence()) && ((o1.isLeftAssoc() && o1.getPrecedence() <= operators.get(nextToken.surface).getPrecedence())
|| (o1.getPrecedence() < operators.get(nextToken.surface).getPrecedence()))) || (o1.getPrecedence() < operators.get(nextToken.surface).getPrecedence())))
{ {
@@ -1059,15 +1059,15 @@ public class Expression
{ {
public LazyValue op; public LazyValue op;
public List<ExpressionNode> args; public List<ExpressionNode> args;
public Tokenizer.Token token; public Token token;
public List<Tokenizer.Token> range; public List<Token> range;
/** /**
* The Value representation of the left parenthesis, used for parsing * The Value representation of the left parenthesis, used for parsing
* varying numbers of function parameters. * varying numbers of function parameters.
*/ */
public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Tokenizer.Token.NONE); public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Token.NONE);
public ExpressionNode(LazyValue op, List<ExpressionNode> args, Tokenizer.Token token) public ExpressionNode(LazyValue op, List<ExpressionNode> args, Token token)
{ {
this.op = op; this.op = op;
this.args = args; this.args = args;
@@ -1076,17 +1076,17 @@ public class Expression
range.add(token); range.add(token);
} }
public static ExpressionNode ofConstant(Value val, Tokenizer.Token token) public static ExpressionNode ofConstant(Value val, Token token)
{ {
return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token); return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token);
} }
} }
private ExpressionNode RPNToParseTree(List<Tokenizer.Token> tokens, Context context) private ExpressionNode RPNToParseTree(List<Token> tokens, Context context)
{ {
Stack<ExpressionNode> nodeStack = new ObjectArrayList<>(); Stack<ExpressionNode> nodeStack = new ObjectArrayList<>();
for (Tokenizer.Token token : tokens) for (Token token : tokens)
{ {
switch (token.type) { switch (token.type) {
case UNARY_OPERATOR -> { case UNARY_OPERATOR -> {
@@ -1104,7 +1104,7 @@ public class Expression
Value constant = getConstantFor(token.surface); Value constant = getConstantFor(token.surface);
if (constant != null) if (constant != null)
{ {
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token)); nodeStack.push(new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token));
} }
else else
@@ -1135,8 +1135,8 @@ public class Expression
} }
if (!isKnown) if (!isKnown)
{ {
p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Tokenizer.Token.TokenType.STRINGPARAM, token.surface))); p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Token.TokenType.STRINGPARAM, token.surface)));
token.morph(Tokenizer.Token.TokenType.FUNCTION, "call"); token.morph(Token.TokenType.FUNCTION, "call");
} }
Collections.reverse(p); Collections.reverse(p);
if (nodeStack.top() == ExpressionNode.PARAMS_START) if (nodeStack.top() == ExpressionNode.PARAMS_START)
@@ -1160,11 +1160,11 @@ public class Expression
{ {
throw new ExpressionException(context, this, token, "Not a number"); throw new ExpressionException(context, this, token, "Not a number");
} }
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(number, token)); nodeStack.push(ExpressionNode.ofConstant(number, token));
} }
case STRINGPARAM -> { case STRINGPARAM -> {
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(new StringValue(token.surface), token)); nodeStack.push(ExpressionNode.ofConstant(new StringValue(token.surface), token));
} }
case HEX_LITERAL -> { case HEX_LITERAL -> {
@@ -1177,7 +1177,7 @@ public class Expression
{ {
throw new ExpressionException(context, this, token, "Not a number"); throw new ExpressionException(context, this, token, "Not a number");
} }
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface); token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(hexNumber, token)); nodeStack.push(ExpressionNode.ofConstant(hexNumber, token));
} }
default -> throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'"); default -> throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'");
@@ -1188,7 +1188,7 @@ public class Expression
private LazyValue getAST(Context context) private LazyValue getAST(Context context)
{ {
List<Tokenizer.Token> rpn = shuntingYard(context); List<Token> rpn = shuntingYard(context);
validate(context, rpn); validate(context, rpn);
ExpressionNode root = RPNToParseTree(rpn, context); ExpressionNode root = RPNToParseTree(rpn, context);
if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server)) if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server))
@@ -1272,7 +1272,7 @@ public class Expression
{ {
// ctx is just to report errors, not values evaluation // ctx is just to report errors, not values evaluation
boolean optimized = false; boolean optimized = false;
Tokenizer.Token.TokenType token = node.token.type; Token.TokenType token = node.token.type;
if (!token.isFunctional()) if (!token.isFunctional())
{ {
return false; return false;
@@ -1284,7 +1284,7 @@ public class Expression
} }
// function or operator // function or operator
String symbol = node.token.surface; String symbol = node.token.surface;
Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol); Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Context.Type requestedType = operation.staticType(expectedType); Context.Type requestedType = operation.staticType(expectedType);
for (ExpressionNode arg : node.args) for (ExpressionNode arg : node.args)
{ {
@@ -1326,7 +1326,7 @@ public class Expression
else else
{ {
returnNode.op = LazyValue.ofConstant(Value.NULL); returnNode.op = LazyValue.ofConstant(Value.NULL);
returnNode.token.morph(Tokenizer.Token.TokenType.CONSTANT, ""); returnNode.token.morph(Token.TokenType.CONSTANT, "");
returnNode.args = Collections.emptyList(); returnNode.args = Collections.emptyList();
if (scriptsDebugging) if (scriptsDebugging)
{ {
@@ -1370,7 +1370,7 @@ public class Expression
{ {
CarpetScriptServer.LOG.info(" - " + symbol + "(" + node.args.size() + ") => " + function + "(" + newargs.size() + ") at line " + (node.token.lineno + 1) + ", node depth " + indent); CarpetScriptServer.LOG.info(" - " + symbol + "(" + node.args.size() + ") => " + function + "(" + newargs.size() + ") at line " + (node.token.lineno + 1) + ", node depth " + indent);
} }
node.token.morph(Tokenizer.Token.TokenType.FUNCTION, function); node.token.morph(Token.TokenType.FUNCTION, function);
node.args = newargs; node.args = newargs;
} }
} }
@@ -1382,7 +1382,7 @@ public class Expression
{ {
// ctx is just to report errors, not values evaluation // ctx is just to report errors, not values evaluation
boolean optimized = false; boolean optimized = false;
Tokenizer.Token.TokenType token = node.token.type; Token.TokenType token = node.token.type;
if (!token.isFunctional()) if (!token.isFunctional())
{ {
return false; return false;
@@ -1396,7 +1396,7 @@ public class Expression
} }
// function or operator // function or operator
Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol); Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Context.Type requestedType = operation.staticType(expectedType); Context.Type requestedType = operation.staticType(expectedType);
for (ExpressionNode arg : node.args) for (ExpressionNode arg : node.args)
{ {
@@ -1491,7 +1491,7 @@ public class Expression
Value ret = ((LazyValue.ContextFreeLazyValue) node.op).evalType(expectedType); Value ret = ((LazyValue.ContextFreeLazyValue) node.op).evalType(expectedType);
return (c, t) -> ret; return (c, t) -> ret;
} }
Tokenizer.Token token = node.token; Token token = node.token;
switch (token.type) switch (token.type)
{ {
case UNARY_OPERATOR: case UNARY_OPERATOR:
@@ -1526,7 +1526,7 @@ public class Expression
} }
} }
private void validate(Context c, List<Tokenizer.Token> rpn) private void validate(Context c, List<Token> rpn)
{ {
/*- /*-
* Thanks to Norman Ramsey: * Thanks to Norman Ramsey:
@@ -1541,7 +1541,7 @@ public class Expression
// push the 'global' scope // push the 'global' scope
stack.push(0); stack.push(0);
for (Tokenizer.Token token : rpn) for (Token token : rpn)
{ {
switch (token.type) switch (token.type)
{ {
src/main/java/carpet/script/Fluff.java
+5 -5
View File
@@ -70,7 +70,7 @@ public abstract class Fluff
boolean numParamsVaries(); boolean numParamsVaries();
LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List<LazyValue> lazyParams); LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List<LazyValue> lazyParams);
static void checkInterrupts() static void checkInterrupts()
{ {
@@ -93,7 +93,7 @@ public abstract class Fluff
boolean isLeftAssoc(); boolean isLeftAssoc();
LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2); LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2);
} }
public interface IOperator extends ILazyOperator public interface IOperator extends ILazyOperator
@@ -187,7 +187,7 @@ public abstract class Fluff
} }
@Override @Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, List<LazyValue> lazyParams)
{ {
return new LazyValue() return new LazyValue()
@@ -273,7 +273,7 @@ public abstract class Fluff
} }
@Override @Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{ {
return (c, typeIgnored) -> { return (c, typeIgnored) -> {
try try
@@ -308,7 +308,7 @@ public abstract class Fluff
} }
@Override @Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{ {
if (v2 != null) if (v2 != null)
{ {
src/main/java/carpet/script/ScriptCommand.java
+1 -3
View File
@@ -37,9 +37,7 @@ import net.minecraft.commands.arguments.blocks.BlockStateArgument;
import net.minecraft.commands.arguments.coordinates.BlockPosArgument; import net.minecraft.commands.arguments.coordinates.BlockPosArgument;
import net.minecraft.core.BlockPos; import net.minecraft.core.BlockPos;
import net.minecraft.server.level.ServerLevel; import net.minecraft.server.level.ServerLevel;
import net.minecraft.world.Clearable;
import net.minecraft.world.level.block.Block; import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.entity.BlockEntity;
import net.minecraft.world.level.block.state.pattern.BlockInWorld; import net.minecraft.world.level.block.state.pattern.BlockInWorld;
import net.minecraft.world.level.levelgen.structure.BoundingBox; import net.minecraft.world.level.levelgen.structure.BoundingBox;
@@ -438,7 +436,7 @@ public class ScriptCommand
return; return;
} }
Expression expr = fun.getExpression(); Expression expr = fun.getExpression();
Tokenizer.Token tok = fun.getToken(); Token tok = fun.getToken();
List<String> snippet = expr.getExpressionSnippet(tok); List<String> snippet = expr.getExpressionSnippet(tok);
Carpet.Messenger_message(source, "wb " + fun.fullName(), "t defined at: line " + (tok.lineno + 1) + " pos " + (tok.linepos + 1)); Carpet.Messenger_message(source, "wb " + fun.fullName(), "t defined at: line " + (tok.lineno + 1) + " pos " + (tok.linepos + 1));
for (String snippetLine : snippet) for (String snippetLine : snippet)
src/main/java/carpet/script/ScriptHost.java
+1 -1
View File
@@ -138,7 +138,7 @@ public abstract class ScriptHost
@FunctionalInterface @FunctionalInterface
public interface ErrorSnooper public interface ErrorSnooper
{ {
List<String> apply(Expression expression, Tokenizer.Token token, Context context, String message); List<String> apply(Expression expression, Token token, Context context, String message);
} }
public ErrorSnooper errorSnooper = null; public ErrorSnooper errorSnooper = null;
src/main/java/carpet/script/Token.java
+69
View File
@@ -0,0 +1,69 @@
package carpet.script;
public class Token {
enum TokenType {
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
VARIABLE(false, false), CONSTANT(false, true),
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
final boolean functional;
final boolean constant;
TokenType(boolean functional, boolean constant) {
this.functional = functional;
this.constant = constant;
}
public boolean isFunctional() {
return functional;
}
public boolean isConstant() {
return constant;
}
}
public String surface = "";
public TokenType type;
public int pos;
public int linepos;
public int lineno;
public static final Token NONE = new Token();
public Token morphedInto(TokenType newType, String newSurface) {
Token created = new Token();
created.surface = newSurface;
created.type = newType;
created.pos = pos;
created.linepos = linepos;
created.lineno = lineno;
return created;
}
public void morph(TokenType type, String s) {
this.type = type;
this.surface = s;
}
public void append(char c) {
surface += c;
}
public void append(String s) {
surface += s;
}
public char charAt(int pos) {
return surface.charAt(pos);
}
public int length() {
return surface.length();
}
@Override
public String toString() {
return surface;
}
}
src/main/java/carpet/script/Tokenizer.java
+1 -80
View File
@@ -14,7 +14,7 @@ import java.util.stream.StreamSupport;
* Expression tokenizer that allows to iterate over a {@link String} * Expression tokenizer that allows to iterate over a {@link String}
* expression token by token. Blank characters will be skipped. * expression token by token. Blank characters will be skipped.
*/ */
public class Tokenizer implements Iterator<Tokenizer.Token> public class Tokenizer implements Iterator<Token>
{ {
/** /**
* What character to use for decimal separators. * What character to use for decimal separators.
@@ -421,83 +421,4 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
throw new InternalExpressionException("remove() not supported"); throw new InternalExpressionException("remove() not supported");
} }
public static class Token
{
enum TokenType
{
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
VARIABLE(false, false), CONSTANT(false, true),
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
final boolean functional;
final boolean constant;
TokenType(boolean functional, boolean constant)
{
this.functional = functional;
this.constant = constant;
}
public boolean isFunctional()
{
return functional;
}
public boolean isConstant()
{
return constant;
}
}
public String surface = "";
public TokenType type;
public int pos;
public int linepos;
public int lineno;
public static final Token NONE = new Token();
public Token morphedInto(TokenType newType, String newSurface)
{
Token created = new Token();
created.surface = newSurface;
created.type = newType;
created.pos = pos;
created.linepos = linepos;
created.lineno = lineno;
return created;
}
public void morph(TokenType type, String s)
{
this.type = type;
this.surface = s;
}
public void append(char c)
{
surface += c;
}
public void append(String s)
{
surface += s;
}
public char charAt(int pos)
{
return surface.charAt(pos);
}
public int length()
{
return surface.length();
}
@Override
public String toString()
{
return surface;
}
}
} }
src/main/java/carpet/script/exception/ExpressionException.java
+8 -8
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context; import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Tokenizer; import carpet.script.Token;
import carpet.script.external.Carpet; import carpet.script.external.Carpet;
import carpet.script.value.FunctionValue; import carpet.script.value.FunctionValue;
@@ -16,7 +16,7 @@ import java.util.function.Supplier;
public class ExpressionException extends StacklessRuntimeException implements ResolvedException public class ExpressionException extends StacklessRuntimeException implements ResolvedException
{ {
public final Context context; public final Context context;
public final Tokenizer.Token token; public final Token token;
public final List<FunctionValue> stack = new ArrayList<>(); public final List<FunctionValue> stack = new ArrayList<>();
private final Supplier<String> lazyMessage; private final Supplier<String> lazyMessage;
private String cachedMessage = null; private String cachedMessage = null;
@@ -28,15 +28,15 @@ public class ExpressionException extends StacklessRuntimeException implements Re
public ExpressionException(Context c, Expression e, String message) public ExpressionException(Context c, Expression e, String message)
{ {
this(c, e, Tokenizer.Token.NONE, message); this(c, e, Token.NONE, message);
} }
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message) public ExpressionException(Context c, Expression e, Token t, String message)
{ {
this(c, e, t, message, Collections.emptyList()); this(c, e, t, message, Collections.emptyList());
} }
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message, List<FunctionValue> stack) public ExpressionException(Context c, Expression e, Token t, String message, List<FunctionValue> stack)
{ {
super("Error"); super("Error");
this.stack.addAll(stack); this.stack.addAll(stack);
@@ -45,7 +45,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
context = c; context = c;
} }
public ExpressionException(Context c, Expression e, Tokenizer.Token t, Supplier<String> messageSupplier, List<FunctionValue> stack) public ExpressionException(Context c, Expression e, Token t, Supplier<String> messageSupplier, List<FunctionValue> stack)
{ {
super("Error"); super("Error");
this.stack.addAll(stack); this.stack.addAll(stack);
@@ -54,7 +54,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
context = c; context = c;
} }
private static List<String> makeError(Expression expr, @Nullable Tokenizer.Token token, String errmessage) private static List<String> makeError(Expression expr, @Nullable Token token, String errmessage)
{ {
List<String> errMsg = new ArrayList<>(); List<String> errMsg = new ArrayList<>();
errmessage += expr.getModuleName() == null ? "" : (" in " + expr.getModuleName()); errmessage += expr.getModuleName() == null ? "" : (" in " + expr.getModuleName());
@@ -76,7 +76,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
return errMsg; return errMsg;
} }
static synchronized String makeMessage(Context c, Expression e, Tokenizer.Token t, String message) throws ExpressionException static synchronized String makeMessage(Context c, Expression e, Token t, String message) throws ExpressionException
{ {
if (c.getErrorSnooper() != null) if (c.getErrorSnooper() != null)
{ {
src/main/java/carpet/script/exception/InternalExpressionException.java
+2 -2
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context; import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Tokenizer; import carpet.script.Token;
import carpet.script.value.FunctionValue; import carpet.script.value.FunctionValue;
import java.util.ArrayList; import java.util.ArrayList;
@@ -30,7 +30,7 @@ public class InternalExpressionException extends StacklessRuntimeException
* @return The new {@link ExpressionException} (or {@link ProcessedThrowStatement}), * @return The new {@link ExpressionException} (or {@link ProcessedThrowStatement}),
* depending on the implementation. * depending on the implementation.
*/ */
public ExpressionException promote(Context c, Expression e, Tokenizer.Token token) public ExpressionException promote(Context c, Expression e, Token token)
{ {
return new ExpressionException(c, e, token, getMessage(), stack); return new ExpressionException(c, e, token, getMessage(), stack);
} }
src/main/java/carpet/script/exception/ProcessedThrowStatement.java
+1 -1
View File
@@ -4,7 +4,7 @@ import java.util.List;
import carpet.script.Context; import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Tokenizer.Token; import carpet.script.Token;
import carpet.script.value.FunctionValue; import carpet.script.value.FunctionValue;
import carpet.script.value.Value; import carpet.script.value.Value;
src/main/java/carpet/script/exception/ThrowStatement.java
+1 -1
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context; import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Tokenizer.Token; import carpet.script.Token;
import carpet.script.value.StringValue; import carpet.script.value.StringValue;
import carpet.script.value.Value; import carpet.script.value.Value;
src/main/java/carpet/script/language/Functions.java
+2 -2
View File
@@ -4,7 +4,7 @@ import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Fluff; import carpet.script.Fluff;
import carpet.script.LazyValue; import carpet.script.LazyValue;
import carpet.script.Tokenizer; import carpet.script.Token;
import carpet.script.argument.FunctionArgument; import carpet.script.argument.FunctionArgument;
import carpet.script.exception.InternalExpressionException; import carpet.script.exception.InternalExpressionException;
import carpet.script.exception.ReturnStatement; import carpet.script.exception.ReturnStatement;
@@ -45,7 +45,7 @@ public class Functions
expression.addCustomFunction("call", new Fluff.AbstractLazyFunction(-1, "call") expression.addCustomFunction("call", new Fluff.AbstractLazyFunction(-1, "call")
{ {
@Override @Override
public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Tokenizer.Token tok, List<LazyValue> lv) public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Token tok, List<LazyValue> lv)
{ {
if (lv.isEmpty()) if (lv.isEmpty())
{ {
src/main/java/carpet/script/value/FunctionValue.java
+7 -7
View File
@@ -6,7 +6,7 @@ import carpet.script.Expression;
import carpet.script.Fluff; import carpet.script.Fluff;
import carpet.script.LazyValue; import carpet.script.LazyValue;
import carpet.script.Module; import carpet.script.Module;
import carpet.script.Tokenizer; import carpet.script.Token;
import carpet.script.exception.BreakStatement; import carpet.script.exception.BreakStatement;
import carpet.script.exception.ContinueStatement; import carpet.script.exception.ContinueStatement;
import carpet.script.exception.ExpressionException; import carpet.script.exception.ExpressionException;
@@ -28,7 +28,7 @@ import javax.annotation.Nullable;
public class FunctionValue extends Value implements Fluff.ILazyFunction public class FunctionValue extends Value implements Fluff.ILazyFunction
{ {
private final Expression expression; private final Expression expression;
private final Tokenizer.Token token; private final Token token;
private final String name; private final String name;
private final LazyValue body; private final LazyValue body;
private Map<String, LazyValue> outerState; private Map<String, LazyValue> outerState;
@@ -37,7 +37,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
private static long variantCounter = 1; private static long variantCounter = 1;
private long variant; private long variant;
private FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List<String> args, String varArgs) private FunctionValue(Expression expression, Token token, String name, LazyValue body, List<String> args, String varArgs)
{ {
this.expression = expression; this.expression = expression;
this.token = token; this.token = token;
@@ -49,7 +49,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
variant = 0L; variant = 0L;
} }
public FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List<String> args, String varArgs, Map<String, LazyValue> outerState) public FunctionValue(Expression expression, Token token, String name, LazyValue body, List<String> args, String varArgs, Map<String, LazyValue> outerState)
{ {
this.expression = expression; this.expression = expression;
this.token = token; this.token = token;
@@ -214,13 +214,13 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
} }
@Override @Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams) public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lazyParams)
{ {
List<Value> resolvedParams = unpackArgs(lazyParams, c); List<Value> resolvedParams = unpackArgs(lazyParams, c);
return execute(c, type, e, t, resolvedParams, null); return execute(c, type, e, t, resolvedParams, null);
} }
public LazyValue execute(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<Value> params, @Nullable ThreadValue freshNewCallingThread) public LazyValue execute(Context c, Context.Type type, Expression e, Token t, List<Value> params, @Nullable ThreadValue freshNewCallingThread)
{ {
assertArgsOk(params, fixedArgs -> { assertArgsOk(params, fixedArgs -> {
if (fixedArgs) // wrong number of args for fixed args if (fixedArgs) // wrong number of args for fixed args
@@ -288,7 +288,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
return expression; return expression;
} }
public Tokenizer.Token getToken() public Token getToken()
{ {
return token; return token;
} }
src/main/java/carpet/script/value/ThreadValue.java
+3 -3
View File
@@ -2,7 +2,7 @@ package carpet.script.value;
import carpet.script.Context; import carpet.script.Context;
import carpet.script.Expression; import carpet.script.Expression;
import carpet.script.Tokenizer; import carpet.script.Token;
import carpet.script.exception.ExitStatement; import carpet.script.exception.ExitStatement;
import carpet.script.exception.ExpressionException; import carpet.script.exception.ExpressionException;
import carpet.script.exception.InternalExpressionException; import carpet.script.exception.InternalExpressionException;
@@ -28,7 +28,7 @@ public class ThreadValue extends LazyListValue
private final AtomicReference<Value> coLock = new AtomicReference<>(Value.EOL); private final AtomicReference<Value> coLock = new AtomicReference<>(Value.EOL);
public final boolean isCoroutine; public final boolean isCoroutine;
public ThreadValue(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List<Value> args) public ThreadValue(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List<Value> args)
{ {
this.id = sequence++; this.id = sequence++;
this.isCoroutine = ctx.host.canSynchronouslyExecute(); this.isCoroutine = ctx.host.canSynchronouslyExecute();
@@ -37,7 +37,7 @@ public class ThreadValue extends LazyListValue
Thread.yield(); Thread.yield();
} }
public CompletableFuture<Value> getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List<Value> args) public CompletableFuture<Value> getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List<Value> args)
{ {
ExecutorService executor = ctx.host.getExecutor(pool); ExecutorService executor = ctx.host.getExecutor(pool);
ThreadValue callingThread = isCoroutine ? this : null; ThreadValue callingThread = isCoroutine ? this : null;