icybear/fabric-carpet
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

extract commonly used class from the Tokenizer

Browse Source
This commit is contained in:
gnembon committed 2025-02-16 18:57:19 +01:00
1 parent 9086db9494
commit 14be0aef52
14 files changed
+172 -184
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main/java/carpet/script/CarpetScriptHost.java
+1 -1
View File
@@ -754,7 +754,7 @@ public class CarpetScriptHost extends ScriptHost
}
}
String sign = "";
for (Tokenizer.Token tok : Tokenizer.simplepass(arg))
for (Token tok : Tokenizer.simplepass(arg))
{
switch (tok.type)
{
src/main/java/carpet/script/Expression.java
+70 -70
View File
@@ -130,7 +130,7 @@ public class Expression
return constants.get(surface);
}
public List<String> getExpressionSnippet(Tokenizer.Token token)
public List<String> getExpressionSnippet(Token token)
{
String code = this.getCodeString();
List<String> output = new ArrayList<>(getExpressionSnippetLeftContext(token, code, 1));
@@ -140,7 +140,7 @@ public class Expression
return output;
}
private static List<String> getExpressionSnippetLeftContext(Tokenizer.Token token, String expr, int contextsize)
private static List<String> getExpressionSnippetLeftContext(Token token, String expr, int contextsize)
{
List<String> output = new ArrayList<>();
String[] lines = expr.split("\n");
@@ -156,7 +156,7 @@ public class Expression
return output;
}
private static List<String> getExpressionSnippetContext(Tokenizer.Token token, String expr)
private static List<String> getExpressionSnippetContext(Token token, String expr)
{
List<String> output = new ArrayList<>();
String[] lines = expr.split("\n");
@@ -173,7 +173,7 @@ public class Expression
return output;
}
private static List<String> getExpressionSnippetRightContext(Tokenizer.Token token, String expr, int contextsize)
private static List<String> getExpressionSnippetRightContext(Token token, String expr, int contextsize)
{
List<String> output = new ArrayList<>();
String[] lines = expr.split("\n");
@@ -213,7 +213,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v, LazyValue v2)
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v, LazyValue v2)
{
try
{
@@ -229,7 +229,7 @@ public class Expression
public void addLazyBinaryOperatorWithDelegation(String surface, int precedence, boolean leftAssoc, boolean pure,
SexFunction<Context, Context.Type, Expression, Tokenizer.Token, LazyValue, LazyValue, LazyValue> lazyfun)
SexFunction<Context, Context.Type, Expression, Token, LazyValue, LazyValue, LazyValue> lazyfun)
{
operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc)
{
@@ -246,7 +246,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2)
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{
try
{
@@ -266,7 +266,7 @@ public class Expression
}
public void addLazyFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive,
QuinnFunction<Context, Context.Type, Expression, Tokenizer.Token, List<LazyValue>, LazyValue> lazyfun)
QuinnFunction<Context, Context.Type, Expression, Token, List<LazyValue>, LazyValue> lazyfun)
{
functions.put(name, new AbstractLazyFunction(numpar, name)
{
@@ -283,7 +283,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lv)
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lv)
{
ILazyFunction.checkInterrupts();
try
@@ -299,7 +299,7 @@ public class Expression
}
public void addFunctionWithDelegation(String name, int numpar, boolean pure, boolean transitive,
QuinnFunction<Context, Context.Type, Expression, Tokenizer.Token, List<Value>, Value> fun)
QuinnFunction<Context, Context.Type, Expression, Token, List<Value>, Value> fun)
{
functions.put(name, new AbstractLazyFunction(numpar, name)
{
@@ -316,7 +316,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lv)
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lv)
{
try
{
@@ -356,7 +356,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2)
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2)
{
ILazyFunction.checkInterrupts();
try
@@ -389,7 +389,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2)
public LazyValue lazyEval(Context c, Context.Type t, Expression e, Token token, LazyValue v1, LazyValue v2)
{
try
{
@@ -404,7 +404,7 @@ public class Expression
});
}
public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Tokenizer.Token token)
public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Token token)
{
if (exc instanceof ExitStatement)
{
@@ -568,7 +568,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{
ILazyFunction.checkInterrupts();
if (numParams >= 0 && lazyParams.size() != numParams)
@@ -606,7 +606,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{
ILazyFunction.checkInterrupts();
try
@@ -644,7 +644,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{
ILazyFunction.checkInterrupts();
try
@@ -676,7 +676,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{
ILazyFunction.checkInterrupts();
try
@@ -715,7 +715,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type i, Expression e, Token t, List<LazyValue> lazyParams)
{
try
{
@@ -730,7 +730,7 @@ public class Expression
});
}
public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Tokenizer.Token token, List<String> arguments, String varArgs, List<String> outers, LazyValue code)
public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Token token, List<String> arguments, String varArgs, List<String> outers, LazyValue code)
{
if (functions.containsKey(name))
{
@@ -799,7 +799,7 @@ public class Expression
}
@Override
public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List<LazyValue> lazyParams)
{
c.host.issueDeprecation(copy + "(...)");
return originalFunction.lazyEval(c, type, expr, token, lazyParams);
@@ -860,18 +860,18 @@ public class Expression
}
private List<Tokenizer.Token> shuntingYard(Context c)
private List<Token> shuntingYard(Context c)
{
List<Tokenizer.Token> outputQueue = new ArrayList<>();
Stack<Tokenizer.Token> stack = new ObjectArrayList<>();
List<Token> outputQueue = new ArrayList<>();
Stack<Token> stack = new ObjectArrayList<>();
Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions);
// stripping lousy but acceptable semicolons
List<Tokenizer.Token> cleanedTokens = tokenizer.postProcess();
List<Token> cleanedTokens = tokenizer.postProcess();
Tokenizer.Token lastFunction = null;
Tokenizer.Token previousToken = null;
for (Tokenizer.Token token : cleanedTokens)
Token lastFunction = null;
Token previousToken = null;
for (Token token : cleanedTokens)
{
switch (token.type)
{
@@ -880,9 +880,9 @@ public class Expression
//break;
case LITERAL, HEX_LITERAL:
if (previousToken != null && (
previousToken.type == Tokenizer.Token.TokenType.LITERAL ||
previousToken.type == Tokenizer.Token.TokenType.HEX_LITERAL ||
previousToken.type == Tokenizer.Token.TokenType.STRINGPARAM))
previousToken.type == Token.TokenType.LITERAL ||
previousToken.type == Token.TokenType.HEX_LITERAL ||
previousToken.type == Token.TokenType.STRINGPARAM))
{
throw new ExpressionException(c, this, token, "Missing operator");
}
@@ -896,11 +896,11 @@ public class Expression
lastFunction = token;
break;
case COMMA:
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR)
if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR)
{
throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator ");
}
while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN)
while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN)
{
outputQueue.add(stack.pop());
}
@@ -919,7 +919,7 @@ public class Expression
case OPERATOR:
{
if (previousToken != null
&& (previousToken.type == Tokenizer.Token.TokenType.COMMA || previousToken.type == Tokenizer.Token.TokenType.OPEN_PAREN))
&& (previousToken.type == Token.TokenType.COMMA || previousToken.type == Token.TokenType.OPEN_PAREN))
{
throw new ExpressionException(c, this, token, "Missing parameter(s) for operator '" + token + "'");
}
@@ -935,8 +935,8 @@ public class Expression
}
case UNARY_OPERATOR:
{
if (previousToken != null && previousToken.type != Tokenizer.Token.TokenType.OPERATOR
&& previousToken.type != Tokenizer.Token.TokenType.COMMA && previousToken.type != Tokenizer.Token.TokenType.OPEN_PAREN)
if (previousToken != null && previousToken.type != Token.TokenType.OPERATOR
&& previousToken.type != Token.TokenType.COMMA && previousToken.type != Token.TokenType.OPEN_PAREN)
{
throw new ExpressionException(c, this, token, "Invalid position for unary operator " + token);
}
@@ -952,18 +952,18 @@ public class Expression
}
case OPEN_PAREN:
// removed implicit multiplication in this missing code block
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.FUNCTION)
if (previousToken != null && previousToken.type == Token.TokenType.FUNCTION)
{
outputQueue.add(token);
}
stack.push(token);
break;
case CLOSE_PAREN:
if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR)
if (previousToken != null && previousToken.type == Token.TokenType.OPERATOR)
{
throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator " + previousToken);
}
while (!stack.isEmpty() && stack.top().type != Tokenizer.Token.TokenType.OPEN_PAREN)
while (!stack.isEmpty() && stack.top().type != Token.TokenType.OPEN_PAREN)
{
outputQueue.add(stack.pop());
}
@@ -972,7 +972,7 @@ public class Expression
throw new ExpressionException(c, this, "Mismatched parentheses");
}
stack.pop();
if (!stack.isEmpty() && stack.top().type == Tokenizer.Token.TokenType.FUNCTION)
if (!stack.isEmpty() && stack.top().type == Token.TokenType.FUNCTION)
{
outputQueue.add(stack.pop());
}
@@ -986,7 +986,7 @@ public class Expression
}
break;
}
if (token.type != Tokenizer.Token.TokenType.MARKER)
if (token.type != Token.TokenType.MARKER)
{
previousToken = token;
}
@@ -994,8 +994,8 @@ public class Expression
while (!stack.isEmpty())
{
Tokenizer.Token element = stack.pop();
if (element.type == Tokenizer.Token.TokenType.OPEN_PAREN || element.type == Tokenizer.Token.TokenType.CLOSE_PAREN)
Token element = stack.pop();
if (element.type == Token.TokenType.OPEN_PAREN || element.type == Token.TokenType.CLOSE_PAREN)
{
throw new ExpressionException(c, this, element, "Mismatched parentheses");
}
@@ -1004,12 +1004,12 @@ public class Expression
return outputQueue;
}
private void shuntOperators(List<Tokenizer.Token> outputQueue, Stack<Tokenizer.Token> stack, ILazyOperator o1)
private void shuntOperators(List<Token> outputQueue, Stack<Token> stack, ILazyOperator o1)
{
Tokenizer.Token nextToken = stack.isEmpty() ? null : stack.top();
Token nextToken = stack.isEmpty() ? null : stack.top();
while (nextToken != null
&& (nextToken.type == Tokenizer.Token.TokenType.OPERATOR
|| nextToken.type == Tokenizer.Token.TokenType.UNARY_OPERATOR)
&& (nextToken.type == Token.TokenType.OPERATOR
|| nextToken.type == Token.TokenType.UNARY_OPERATOR)
&& ((o1.isLeftAssoc() && o1.getPrecedence() <= operators.get(nextToken.surface).getPrecedence())
|| (o1.getPrecedence() < operators.get(nextToken.surface).getPrecedence())))
{
@@ -1059,15 +1059,15 @@ public class Expression
{
public LazyValue op;
public List<ExpressionNode> args;
public Tokenizer.Token token;
public List<Tokenizer.Token> range;
public Token token;
public List<Token> range;
/**
* The Value representation of the left parenthesis, used for parsing
* varying numbers of function parameters.
*/
public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Tokenizer.Token.NONE);
public static final ExpressionNode PARAMS_START = new ExpressionNode(null, null, Token.NONE);
public ExpressionNode(LazyValue op, List<ExpressionNode> args, Tokenizer.Token token)
public ExpressionNode(LazyValue op, List<ExpressionNode> args, Token token)
{
this.op = op;
this.args = args;
@@ -1076,17 +1076,17 @@ public class Expression
range.add(token);
}
public static ExpressionNode ofConstant(Value val, Tokenizer.Token token)
public static ExpressionNode ofConstant(Value val, Token token)
{
return new ExpressionNode(new LazyValue.Constant(val), Collections.emptyList(), token);
}
}
private ExpressionNode RPNToParseTree(List<Tokenizer.Token> tokens, Context context)
private ExpressionNode RPNToParseTree(List<Token> tokens, Context context)
{
Stack<ExpressionNode> nodeStack = new ObjectArrayList<>();
for (Tokenizer.Token token : tokens)
for (Token token : tokens)
{
switch (token.type) {
case UNARY_OPERATOR -> {
@@ -1104,7 +1104,7 @@ public class Expression
Value constant = getConstantFor(token.surface);
if (constant != null)
{
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface);
token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(new ExpressionNode(LazyValue.ofConstant(constant), Collections.emptyList(), token));
}
else
@@ -1135,8 +1135,8 @@ public class Expression
}
if (!isKnown)
{
p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Tokenizer.Token.TokenType.STRINGPARAM, token.surface)));
token.morph(Tokenizer.Token.TokenType.FUNCTION, "call");
p.add(ExpressionNode.ofConstant(new StringValue(name), token.morphedInto(Token.TokenType.STRINGPARAM, token.surface)));
token.morph(Token.TokenType.FUNCTION, "call");
}
Collections.reverse(p);
if (nodeStack.top() == ExpressionNode.PARAMS_START)
@@ -1160,11 +1160,11 @@ public class Expression
{
throw new ExpressionException(context, this, token, "Not a number");
}
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface);
token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(number, token));
}
case STRINGPARAM -> {
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface);
token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(new StringValue(token.surface), token));
}
case HEX_LITERAL -> {
@@ -1177,7 +1177,7 @@ public class Expression
{
throw new ExpressionException(context, this, token, "Not a number");
}
token.morph(Tokenizer.Token.TokenType.CONSTANT, token.surface);
token.morph(Token.TokenType.CONSTANT, token.surface);
nodeStack.push(ExpressionNode.ofConstant(hexNumber, token));
}
default -> throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'");
@@ -1188,7 +1188,7 @@ public class Expression
private LazyValue getAST(Context context)
{
List<Tokenizer.Token> rpn = shuntingYard(context);
List<Token> rpn = shuntingYard(context);
validate(context, rpn);
ExpressionNode root = RPNToParseTree(rpn, context);
if (!Vanilla.ScriptServer_scriptOptimizations(((CarpetScriptServer)context.scriptServer()).server))
@@ -1272,7 +1272,7 @@ public class Expression
{
// ctx is just to report errors, not values evaluation
boolean optimized = false;
Tokenizer.Token.TokenType token = node.token.type;
Token.TokenType token = node.token.type;
if (!token.isFunctional())
{
return false;
@@ -1284,7 +1284,7 @@ public class Expression
}
// function or operator
String symbol = node.token.surface;
Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Context.Type requestedType = operation.staticType(expectedType);
for (ExpressionNode arg : node.args)
{
@@ -1326,7 +1326,7 @@ public class Expression
else
{
returnNode.op = LazyValue.ofConstant(Value.NULL);
returnNode.token.morph(Tokenizer.Token.TokenType.CONSTANT, "");
returnNode.token.morph(Token.TokenType.CONSTANT, "");
returnNode.args = Collections.emptyList();
if (scriptsDebugging)
{
@@ -1370,7 +1370,7 @@ public class Expression
{
CarpetScriptServer.LOG.info(" - " + symbol + "(" + node.args.size() + ") => " + function + "(" + newargs.size() + ") at line " + (node.token.lineno + 1) + ", node depth " + indent);
}
node.token.morph(Tokenizer.Token.TokenType.FUNCTION, function);
node.token.morph(Token.TokenType.FUNCTION, function);
node.args = newargs;
}
}
@@ -1382,7 +1382,7 @@ public class Expression
{
// ctx is just to report errors, not values evaluation
boolean optimized = false;
Tokenizer.Token.TokenType token = node.token.type;
Token.TokenType token = node.token.type;
if (!token.isFunctional())
{
return false;
@@ -1396,7 +1396,7 @@ public class Expression
}
// function or operator
Fluff.EvalNode operation = ((token == Tokenizer.Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Fluff.EvalNode operation = ((token == Token.TokenType.FUNCTION) ? functions : operators).get(symbol);
Context.Type requestedType = operation.staticType(expectedType);
for (ExpressionNode arg : node.args)
{
@@ -1491,7 +1491,7 @@ public class Expression
Value ret = ((LazyValue.ContextFreeLazyValue) node.op).evalType(expectedType);
return (c, t) -> ret;
}
Tokenizer.Token token = node.token;
Token token = node.token;
switch (token.type)
{
case UNARY_OPERATOR:
@@ -1526,7 +1526,7 @@ public class Expression
}
}
private void validate(Context c, List<Tokenizer.Token> rpn)
private void validate(Context c, List<Token> rpn)
{
/*-
* Thanks to Norman Ramsey:
@@ -1541,7 +1541,7 @@ public class Expression
// push the 'global' scope
stack.push(0);
for (Tokenizer.Token token : rpn)
for (Token token : rpn)
{
switch (token.type)
{
src/main/java/carpet/script/Fluff.java
+5 -5
View File
@@ -70,7 +70,7 @@ public abstract class Fluff
boolean numParamsVaries();
LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List<LazyValue> lazyParams);
LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List<LazyValue> lazyParams);
static void checkInterrupts()
{
@@ -93,7 +93,7 @@ public abstract class Fluff
boolean isLeftAssoc();
LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2);
LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2);
}
public interface IOperator extends ILazyOperator
@@ -187,7 +187,7 @@ public abstract class Fluff
}
@Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, List<LazyValue> lazyParams)
{
return new LazyValue()
@@ -273,7 +273,7 @@ public abstract class Fluff
}
@Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2)
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{
return (c, typeIgnored) -> {
try
@@ -308,7 +308,7 @@ public abstract class Fluff
}
@Override
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2)
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
{
if (v2 != null)
{
src/main/java/carpet/script/ScriptCommand.java
+1 -3
View File
@@ -37,9 +37,7 @@ import net.minecraft.commands.arguments.blocks.BlockStateArgument;
import net.minecraft.commands.arguments.coordinates.BlockPosArgument;
import net.minecraft.core.BlockPos;
import net.minecraft.server.level.ServerLevel;
import net.minecraft.world.Clearable;
import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.entity.BlockEntity;
import net.minecraft.world.level.block.state.pattern.BlockInWorld;
import net.minecraft.world.level.levelgen.structure.BoundingBox;
@@ -438,7 +436,7 @@ public class ScriptCommand
return;
}
Expression expr = fun.getExpression();
Tokenizer.Token tok = fun.getToken();
Token tok = fun.getToken();
List<String> snippet = expr.getExpressionSnippet(tok);
Carpet.Messenger_message(source, "wb " + fun.fullName(), "t defined at: line " + (tok.lineno + 1) + " pos " + (tok.linepos + 1));
for (String snippetLine : snippet)
src/main/java/carpet/script/ScriptHost.java
+1 -1
View File
@@ -138,7 +138,7 @@ public abstract class ScriptHost
@FunctionalInterface
public interface ErrorSnooper
{
List<String> apply(Expression expression, Tokenizer.Token token, Context context, String message);
List<String> apply(Expression expression, Token token, Context context, String message);
}
public ErrorSnooper errorSnooper = null;
src/main/java/carpet/script/Token.java
+69
View File
@@ -0,0 +1,69 @@
package carpet.script;
public class Token {
enum TokenType {
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
VARIABLE(false, false), CONSTANT(false, true),
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
final boolean functional;
final boolean constant;
TokenType(boolean functional, boolean constant) {
this.functional = functional;
this.constant = constant;
}
public boolean isFunctional() {
return functional;
}
public boolean isConstant() {
return constant;
}
}
public String surface = "";
public TokenType type;
public int pos;
public int linepos;
public int lineno;
public static final Token NONE = new Token();
public Token morphedInto(TokenType newType, String newSurface) {
Token created = new Token();
created.surface = newSurface;
created.type = newType;
created.pos = pos;
created.linepos = linepos;
created.lineno = lineno;
return created;
}
public void morph(TokenType type, String s) {
this.type = type;
this.surface = s;
}
public void append(char c) {
surface += c;
}
public void append(String s) {
surface += s;
}
public char charAt(int pos) {
return surface.charAt(pos);
}
public int length() {
return surface.length();
}
@Override
public String toString() {
return surface;
}
}
src/main/java/carpet/script/Tokenizer.java
+1 -80
View File
@@ -14,7 +14,7 @@ import java.util.stream.StreamSupport;
* Expression tokenizer that allows to iterate over a {@link String}
* expression token by token. Blank characters will be skipped.
*/
public class Tokenizer implements Iterator<Tokenizer.Token>
public class Tokenizer implements Iterator<Token>
{
/**
* What character to use for decimal separators.
@@ -421,83 +421,4 @@ public class Tokenizer implements Iterator<Tokenizer.Token>
throw new InternalExpressionException("remove() not supported");
}
public static class Token
{
enum TokenType
{
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
VARIABLE(false, false), CONSTANT(false, true),
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
final boolean functional;
final boolean constant;
TokenType(boolean functional, boolean constant)
{
this.functional = functional;
this.constant = constant;
}
public boolean isFunctional()
{
return functional;
}
public boolean isConstant()
{
return constant;
}
}
public String surface = "";
public TokenType type;
public int pos;
public int linepos;
public int lineno;
public static final Token NONE = new Token();
public Token morphedInto(TokenType newType, String newSurface)
{
Token created = new Token();
created.surface = newSurface;
created.type = newType;
created.pos = pos;
created.linepos = linepos;
created.lineno = lineno;
return created;
}
public void morph(TokenType type, String s)
{
this.type = type;
this.surface = s;
}
public void append(char c)
{
surface += c;
}
public void append(String s)
{
surface += s;
}
public char charAt(int pos)
{
return surface.charAt(pos);
}
public int length()
{
return surface.length();
}
@Override
public String toString()
{
return surface;
}
}
}
src/main/java/carpet/script/exception/ExpressionException.java
+8 -8
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Tokenizer;
import carpet.script.Token;
import carpet.script.external.Carpet;
import carpet.script.value.FunctionValue;
@@ -16,7 +16,7 @@ import java.util.function.Supplier;
public class ExpressionException extends StacklessRuntimeException implements ResolvedException
{
public final Context context;
public final Tokenizer.Token token;
public final Token token;
public final List<FunctionValue> stack = new ArrayList<>();
private final Supplier<String> lazyMessage;
private String cachedMessage = null;
@@ -28,15 +28,15 @@ public class ExpressionException extends StacklessRuntimeException implements Re
public ExpressionException(Context c, Expression e, String message)
{
this(c, e, Tokenizer.Token.NONE, message);
this(c, e, Token.NONE, message);
}
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message)
public ExpressionException(Context c, Expression e, Token t, String message)
{
this(c, e, t, message, Collections.emptyList());
}
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message, List<FunctionValue> stack)
public ExpressionException(Context c, Expression e, Token t, String message, List<FunctionValue> stack)
{
super("Error");
this.stack.addAll(stack);
@@ -45,7 +45,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
context = c;
}
public ExpressionException(Context c, Expression e, Tokenizer.Token t, Supplier<String> messageSupplier, List<FunctionValue> stack)
public ExpressionException(Context c, Expression e, Token t, Supplier<String> messageSupplier, List<FunctionValue> stack)
{
super("Error");
this.stack.addAll(stack);
@@ -54,7 +54,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
context = c;
}
private static List<String> makeError(Expression expr, @Nullable Tokenizer.Token token, String errmessage)
private static List<String> makeError(Expression expr, @Nullable Token token, String errmessage)
{
List<String> errMsg = new ArrayList<>();
errmessage += expr.getModuleName() == null ? "" : (" in " + expr.getModuleName());
@@ -76,7 +76,7 @@ public class ExpressionException extends StacklessRuntimeException implements Re
return errMsg;
}
static synchronized String makeMessage(Context c, Expression e, Tokenizer.Token t, String message) throws ExpressionException
static synchronized String makeMessage(Context c, Expression e, Token t, String message) throws ExpressionException
{
if (c.getErrorSnooper() != null)
{
src/main/java/carpet/script/exception/InternalExpressionException.java
+2 -2
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Tokenizer;
import carpet.script.Token;
import carpet.script.value.FunctionValue;
import java.util.ArrayList;
@@ -30,7 +30,7 @@ public class InternalExpressionException extends StacklessRuntimeException
* @return The new {@link ExpressionException} (or {@link ProcessedThrowStatement}),
* depending on the implementation.
*/
public ExpressionException promote(Context c, Expression e, Tokenizer.Token token)
public ExpressionException promote(Context c, Expression e, Token token)
{
return new ExpressionException(c, e, token, getMessage(), stack);
}
src/main/java/carpet/script/exception/ProcessedThrowStatement.java
+1 -1
View File
@@ -4,7 +4,7 @@ import java.util.List;
import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Tokenizer.Token;
import carpet.script.Token;
import carpet.script.value.FunctionValue;
import carpet.script.value.Value;
src/main/java/carpet/script/exception/ThrowStatement.java
+1 -1
View File
@@ -2,7 +2,7 @@ package carpet.script.exception;
import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Tokenizer.Token;
import carpet.script.Token;
import carpet.script.value.StringValue;
import carpet.script.value.Value;
src/main/java/carpet/script/language/Functions.java
+2 -2
View File
@@ -4,7 +4,7 @@ import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Fluff;
import carpet.script.LazyValue;
import carpet.script.Tokenizer;
import carpet.script.Token;
import carpet.script.argument.FunctionArgument;
import carpet.script.exception.InternalExpressionException;
import carpet.script.exception.ReturnStatement;
@@ -45,7 +45,7 @@ public class Functions
expression.addCustomFunction("call", new Fluff.AbstractLazyFunction(-1, "call")
{
@Override
public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Tokenizer.Token tok, List<LazyValue> lv)
public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Token tok, List<LazyValue> lv)
{
if (lv.isEmpty())
{
src/main/java/carpet/script/value/FunctionValue.java
+7 -7
View File
@@ -6,7 +6,7 @@ import carpet.script.Expression;
import carpet.script.Fluff;
import carpet.script.LazyValue;
import carpet.script.Module;
import carpet.script.Tokenizer;
import carpet.script.Token;
import carpet.script.exception.BreakStatement;
import carpet.script.exception.ContinueStatement;
import carpet.script.exception.ExpressionException;
@@ -28,7 +28,7 @@ import javax.annotation.Nullable;
public class FunctionValue extends Value implements Fluff.ILazyFunction
{
private final Expression expression;
private final Tokenizer.Token token;
private final Token token;
private final String name;
private final LazyValue body;
private Map<String, LazyValue> outerState;
@@ -37,7 +37,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
private static long variantCounter = 1;
private long variant;
private FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List<String> args, String varArgs)
private FunctionValue(Expression expression, Token token, String name, LazyValue body, List<String> args, String varArgs)
{
this.expression = expression;
this.token = token;
@@ -49,7 +49,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
variant = 0L;
}
public FunctionValue(Expression expression, Tokenizer.Token token, String name, LazyValue body, List<String> args, String varArgs, Map<String, LazyValue> outerState)
public FunctionValue(Expression expression, Token token, String name, LazyValue body, List<String> args, String varArgs, Map<String, LazyValue> outerState)
{
this.expression = expression;
this.token = token;
@@ -214,13 +214,13 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
}
@Override
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
public LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, List<LazyValue> lazyParams)
{
List<Value> resolvedParams = unpackArgs(lazyParams, c);
return execute(c, type, e, t, resolvedParams, null);
}
public LazyValue execute(Context c, Context.Type type, Expression e, Tokenizer.Token t, List<Value> params, @Nullable ThreadValue freshNewCallingThread)
public LazyValue execute(Context c, Context.Type type, Expression e, Token t, List<Value> params, @Nullable ThreadValue freshNewCallingThread)
{
assertArgsOk(params, fixedArgs -> {
if (fixedArgs) // wrong number of args for fixed args
@@ -288,7 +288,7 @@ public class FunctionValue extends Value implements Fluff.ILazyFunction
return expression;
}
public Tokenizer.Token getToken()
public Token getToken()
{
return token;
}
src/main/java/carpet/script/value/ThreadValue.java
+3 -3
View File
@@ -2,7 +2,7 @@ package carpet.script.value;
import carpet.script.Context;
import carpet.script.Expression;
import carpet.script.Tokenizer;
import carpet.script.Token;
import carpet.script.exception.ExitStatement;
import carpet.script.exception.ExpressionException;
import carpet.script.exception.InternalExpressionException;
@@ -28,7 +28,7 @@ public class ThreadValue extends LazyListValue
private final AtomicReference<Value> coLock = new AtomicReference<>(Value.EOL);
public final boolean isCoroutine;
public ThreadValue(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List<Value> args)
public ThreadValue(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List<Value> args)
{
this.id = sequence++;
this.isCoroutine = ctx.host.canSynchronouslyExecute();
@@ -37,7 +37,7 @@ public class ThreadValue extends LazyListValue
Thread.yield();
}
public CompletableFuture<Value> getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Tokenizer.Token token, Context ctx, List<Value> args)
public CompletableFuture<Value> getCompletableFutureFromFunction(Value pool, FunctionValue function, Expression expr, Token token, Context ctx, List<Value> args)
{
ExecutorService executor = ctx.host.getExecutor(pool);
ThreadValue callingThread = isCoroutine ? this : null;