package carpet.script; import carpet.script.Fluff.AbstractFunction; import carpet.script.Fluff.AbstractLazyFunction; import carpet.script.Fluff.AbstractLazyOperator; import carpet.script.Fluff.AbstractOperator; import carpet.script.Fluff.AbstractUnaryOperator; import carpet.script.Fluff.ILazyFunction; import carpet.script.Fluff.ILazyOperator; import carpet.script.Fluff.QuadFunction; import carpet.script.Fluff.QuinnFunction; import carpet.script.Fluff.SexFunction; import carpet.script.Fluff.TriFunction; import carpet.script.bundled.Module; import carpet.script.exception.BreakStatement; import carpet.script.exception.ContinueStatement; import carpet.script.exception.ExitStatement; import carpet.script.exception.ExpressionException; import carpet.script.exception.InternalExpressionException; import carpet.script.exception.ResolvedException; import carpet.script.exception.ReturnStatement; import carpet.script.language.Arithmetic; import carpet.script.language.ControlFlow; import carpet.script.language.DataStructures; import carpet.script.language.Functions; import carpet.script.language.Loops; import carpet.script.language.Operators; import carpet.script.language.Sys; import carpet.script.language.Threading; import carpet.script.value.FunctionValue; import carpet.script.value.NumericValue; import carpet.script.value.StringValue; import carpet.script.value.Value; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Stack; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; import static java.lang.Math.max; import static java.lang.Math.min; public class Expression { /** The current infix expression */ private String expression; String getCodeString() {return expression;} private boolean allowNewlineSubstitutions = true; private boolean allowComments = false; public Module module = null; public String getModuleName() { return module == null?null:module.getName(); } public void asATextSource() { allowNewlineSubstitutions = false; allowComments = true; } public void asAModule(Module mi) { module = mi; } /** Cached AST (Abstract Syntax Tree) (root) of the expression */ private LazyValue ast = null; /** script specific operatos and built-in functions */ private final Map operators = new Object2ObjectOpenHashMap<>(); public boolean isAnOperator(String opname) { return operators.containsKey(opname) || operators.containsKey(opname+"u");} private final Map functions = new Object2ObjectOpenHashMap<>(); public Set getFunctionNames() {return functions.keySet();} public List getExpressionSnippet(Tokenizer.Token token) { String code = this.getCodeString(); List output = new ArrayList<>(getExpressionSnippetLeftContext(token, code, 1)); List context = getExpressionSnippetContext(token, code); output.add(context.get(0)+" HERE>> "+context.get(1)); output.addAll(getExpressionSnippetRightContext(token, code, 1)); return output; } private static List getExpressionSnippetLeftContext(Tokenizer.Token token, String expr, int contextsize) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); if (lines.length == 1) return output; for (int lno=token.lineno-1; lno >=0 && output.size() < contextsize; lno-- ) { output.add(lines[lno]); } Collections.reverse(output); return output; } private static List getExpressionSnippetContext(Tokenizer.Token token, String expr) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); if (lines.length > 1) { output.add(lines[token.lineno].substring(0, token.linepos)); output.add(lines[token.lineno].substring(token.linepos)); } else { output.add( expr.substring(max(0, token.pos-40), token.pos)); output.add( expr.substring(token.pos, min(token.pos+1+40, expr.length()))); } return output; } private static List getExpressionSnippetRightContext(Tokenizer.Token token, String expr, int contextsize) { List output = new ArrayList<>(); String[] lines = expr.split("\n"); if (lines.length == 1) { return output; } for (int lno=token.lineno+1; lno < lines.length && output.size() < contextsize; lno++ ) { output.add(lines[lno]); } return output; } public void addLazyUnaryOperator(String surface, int precedence, boolean leftAssoc, TriFunction lazyfun) { operators.put(surface+"u", new AbstractLazyOperator(precedence, leftAssoc) { @Override public LazyValue lazyEval(Context c, Integer t, Expression e, Tokenizer.Token token, LazyValue v, LazyValue v2) { try { if (v2 != null) { throw new ExpressionException(c, e, token, "Did not expect a second parameter for unary operator"); } Value.assertNotNull(v); return lazyfun.apply(c, t, v); } catch (RuntimeException exc) { throw handleCodeException(c, exc, e, token); } } }); } public void addLazyBinaryOperatorWithDelegation(String surface, int precedence, boolean leftAssoc, SexFunction lazyfun) { operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc) { @Override public LazyValue lazyEval(Context c, Integer type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2) { try { Value.assertNotNull(v1, v2); return lazyfun.apply(c, type, e, t, v1, v2); } catch (RuntimeException exc) { throw handleCodeException(c, exc, e, t); } } }); } public void addLazyFunctionWithDelegation(String name, int numpar, QuinnFunction, LazyValue> lazyfun) { functions.put(name, new AbstractLazyFunction(numpar) { @Override public LazyValue lazyEval(Context c, Integer type, Expression e, Tokenizer.Token t, List lv) { try { return lazyfun.apply(c, type, e, t, lv); } catch (RuntimeException exc) { throw handleCodeException(c, exc, e, t); } } }); } public void addLazyBinaryOperator(String surface, int precedence, boolean leftAssoc, QuadFunction lazyfun) { operators.put(surface, new AbstractLazyOperator(precedence, leftAssoc) { @Override public LazyValue lazyEval(Context c, Integer t, Expression e, Tokenizer.Token token, LazyValue v1, LazyValue v2) { try { Value.assertNotNull(v1, v2); return lazyfun.apply(c, t, v1, v2); } catch (RuntimeException exc) { throw handleCodeException(c, exc, e, token); } } }); } public static RuntimeException handleCodeException(Context c, RuntimeException exc, Expression e, Tokenizer.Token token) { if (exc instanceof ExitStatement) return exc; if (exc instanceof InternalExpressionException) return ((InternalExpressionException) exc).promote(c, e, token); if (exc instanceof ArithmeticException) return new ExpressionException(c, e, token, "Your math is wrong, "+exc.getMessage()); if (exc instanceof ResolvedException) return exc; // unexpected really - should be caught earlier and converted to InternalExpressionException exc.printStackTrace(); return new ExpressionException(c, e, token, "Error while evaluating expression: "+exc); } public void addUnaryOperator(String surface, boolean leftAssoc, Function fun) { operators.put(surface+"u", new AbstractUnaryOperator(Operators.precedence.get("unary+-!"), leftAssoc) { @Override public Value evalUnary(Value v1) { return fun.apply(Value.assertNotNull(v1)); } }); } public void addBinaryOperator(String surface, int precedence, boolean leftAssoc, BiFunction fun) { operators.put(surface, new AbstractOperator(precedence, leftAssoc) { @Override public Value eval(Value v1, Value v2) { Value.assertNotNull(v1, v2); return fun.apply(v1, v2); } }); } public void addUnaryFunction(String name, Function fun) { functions.put(name, new AbstractFunction(1) { @Override public Value eval(List parameters) { return fun.apply(Value.assertNotNull(parameters.get(0))); } }); } public void addBinaryFunction(String name, BiFunction fun) { functions.put(name, new AbstractFunction(2) { @Override public Value eval(List parameters) { Value v1 = parameters.get(0); Value v2 = parameters.get(1); Value.assertNotNull(v1, v2); return fun.apply(v1, v2); } }); } public void addFunction(String name, Function, Value> fun) { functions.put(name, new AbstractFunction(-1) { @Override public Value eval(List parameters) { for (Value v: parameters) Value.assertNotNull(v); return fun.apply(parameters); } }); } public void addMathematicalUnaryFunction(String name, Function fun) { addUnaryFunction(name, (v) -> new NumericValue(fun.apply(NumericValue.asNumber(v).getDouble()))); } public void addMathematicalUnaryIntFunction(String name, Function fun) { addUnaryFunction(name, (v) -> new NumericValue(fun.apply(NumericValue.asNumber(v).getDouble()))); } public void addMathematicalBinaryFunction(String name, BiFunction fun) { addBinaryFunction(name, (w, v) -> new NumericValue(fun.apply(NumericValue.asNumber(w).getDouble(), NumericValue.asNumber(v).getDouble()))); } public void addLazyFunction(String name, int num_params, TriFunction, LazyValue> fun) { functions.put(name, new AbstractLazyFunction(num_params) { @Override public LazyValue lazyEval(Context c, Integer i, Expression e, Tokenizer.Token t, List lazyParams) { try { return fun.apply(c, i, lazyParams); } catch (RuntimeException exc) { throw handleCodeException(c, exc, e, t); } } }); } public FunctionValue createUserDefinedFunction(Context context, String name, Expression expr, Tokenizer.Token token, List arguments, String varArgs, List outers, LazyValue code) { if (functions.containsKey(name)) throw new ExpressionException(context, expr, token, "Function "+name+" would mask a built-in function"); Map contextValues = new HashMap<>(); for (String outer : outers) { LazyValue lv = context.getVariable(outer); if (lv == null) { throw new InternalExpressionException("Variable "+outer+" needs to be defined in outer scope to be used as outer parameter, and cannot be global"); } else { contextValues.put(outer, lv); } } if (contextValues.isEmpty()) contextValues = null; FunctionValue result = new FunctionValue(expr, token, name, code, arguments, varArgs, contextValues); // do not store lambda definitions if (!name.equals("_")) context.host.addUserDefinedFunction(context, module, name, result); return result; } public void alias(String copy, String original) { ILazyFunction originalFunction = functions.get(original); functions.put(copy, new ILazyFunction() { @Override public int getNumParams() { return originalFunction.getNumParams(); } @Override public boolean numParamsVaries() { return originalFunction.numParamsVaries(); } @Override public LazyValue lazyEval(Context c, Integer type, Expression expr, Tokenizer.Token token, List lazyParams) { c.host.issueDeprecation(copy+"(...)"); return originalFunction.lazyEval(c, type, expr, token, lazyParams); } }); } public void setAnyVariable(Context c, String name, LazyValue lv) { if (name.startsWith("global_")) { c.host.setGlobalVariable(module, name, lv); } else { c.setVariable(name, lv); } } public LazyValue getOrSetAnyVariable(Context c, String name) { LazyValue var = null; if (!name.startsWith("global_")) { var = c.getVariable(name); if (var != null) return var; } var = c.host.getGlobalVariable(module, name); if (var != null) return var; var = (_c, _t ) -> Value.ZERO.reboundedTo(name); setAnyVariable(c, name, var); return var; } public static final Expression none = new Expression("null"); /** * @param expression . */ public Expression(String expression) { this.expression = expression.trim().replaceAll("\\r\\n?", "\n").replaceAll("\\t"," "); Operators.apply(this); ControlFlow.apply(this); Functions.apply(this); Arithmetic.apply(this); Sys.apply(this); Threading.apply(this); Loops.apply(this); DataStructures.apply(this); } private List shuntingYard(Context c) { List outputQueue = new ArrayList<>(); Stack stack = new Stack<>(); Tokenizer tokenizer = new Tokenizer(c, this, expression, allowComments, allowNewlineSubstitutions); // stripping lousy but acceptable semicolons List cleanedTokens = tokenizer.postProcess(); Tokenizer.Token lastFunction = null; Tokenizer.Token previousToken = null; for (Tokenizer.Token token : cleanedTokens) { switch (token.type) { case STRINGPARAM: //stack.push(token); // changed that so strings are treated like literals //break; case LITERAL: case HEX_LITERAL: if (previousToken != null && ( previousToken.type == Tokenizer.Token.TokenType.LITERAL || previousToken.type == Tokenizer.Token.TokenType.HEX_LITERAL || previousToken.type == Tokenizer.Token.TokenType.STRINGPARAM)) { throw new ExpressionException(c, this, token, "Missing operator"); } outputQueue.add(token); break; case VARIABLE: outputQueue.add(token); break; case FUNCTION: stack.push(token); lastFunction = token; break; case COMMA: if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) { throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator "); } while (!stack.isEmpty() && stack.peek().type != Tokenizer.Token.TokenType.OPEN_PAREN) { outputQueue.add(stack.pop()); } if (stack.isEmpty()) { if (lastFunction == null) { throw new ExpressionException(c, this, token, "Unexpected comma"); } else { throw new ExpressionException(c, this, lastFunction, "Parse error for function"); } } break; case OPERATOR: { if (previousToken != null && (previousToken.type == Tokenizer.Token.TokenType.COMMA || previousToken.type == Tokenizer.Token.TokenType.OPEN_PAREN)) { throw new ExpressionException(c, this, token, "Missing parameter(s) for operator '" + token+"'"); } ILazyOperator o1 = operators.get(token.surface); if (o1 == null) { throw new ExpressionException(c, this, token, "Unknown operator '" + token + "'"); } shuntOperators(outputQueue, stack, o1); stack.push(token); break; } case UNARY_OPERATOR: { if (previousToken != null && previousToken.type != Tokenizer.Token.TokenType.OPERATOR && previousToken.type != Tokenizer.Token.TokenType.COMMA && previousToken.type != Tokenizer.Token.TokenType.OPEN_PAREN) { throw new ExpressionException(c, this, token, "Invalid position for unary operator " + token ); } ILazyOperator o1 = operators.get(token.surface); if (o1 == null) { throw new ExpressionException(c, this, token, "Unknown unary operator '" + token.surface.substring(0, token.surface.length() - 1) + "'"); } shuntOperators(outputQueue, stack, o1); stack.push(token); break; } case OPEN_PAREN: if (previousToken != null) { if (previousToken.type == Tokenizer.Token.TokenType.LITERAL || previousToken.type == Tokenizer.Token.TokenType.CLOSE_PAREN || previousToken.type == Tokenizer.Token.TokenType.VARIABLE || previousToken.type == Tokenizer.Token.TokenType.HEX_LITERAL) { // Implicit multiplication, e.g. 23(a+b) or (a+b)(a-b) Tokenizer.Token multiplication = new Tokenizer.Token(); multiplication.append("*"); multiplication.type = Tokenizer.Token.TokenType.OPERATOR; stack.push(multiplication); } // if the ( is preceded by a valid function, then it // denotes the start of a parameter list if (previousToken.type == Tokenizer.Token.TokenType.FUNCTION) { outputQueue.add(token); } } stack.push(token); break; case CLOSE_PAREN: if (previousToken != null && previousToken.type == Tokenizer.Token.TokenType.OPERATOR) { throw new ExpressionException(c, this, previousToken, "Missing parameter(s) for operator " + previousToken); } while (!stack.isEmpty() && stack.peek().type != Tokenizer.Token.TokenType.OPEN_PAREN) { outputQueue.add(stack.pop()); } if (stack.isEmpty()) { throw new ExpressionException(c, this, "Mismatched parentheses"); } stack.pop(); if (!stack.isEmpty() && stack.peek().type == Tokenizer.Token.TokenType.FUNCTION) { outputQueue.add(stack.pop()); } case MARKER: if ("$".equals(token.surface)) { StringBuilder sb = new StringBuilder(expression); sb.setCharAt(token.pos, '\n'); expression = sb.toString(); } break; } if (token.type != Tokenizer.Token.TokenType.MARKER) previousToken = token; } while (!stack.isEmpty()) { Tokenizer.Token element = stack.pop(); if (element.type == Tokenizer.Token.TokenType.OPEN_PAREN || element.type == Tokenizer.Token.TokenType.CLOSE_PAREN) { throw new ExpressionException(c, this, element, "Mismatched parentheses"); } outputQueue.add(element); } return outputQueue; } private void shuntOperators(List outputQueue, Stack stack, ILazyOperator o1) { Tokenizer.Token nextToken = stack.isEmpty() ? null : stack.peek(); while (nextToken != null && (nextToken.type == Tokenizer.Token.TokenType.OPERATOR || nextToken.type == Tokenizer.Token.TokenType.UNARY_OPERATOR) && ((o1.isLeftAssoc() && o1.getPrecedence() <= operators.get(nextToken.surface).getPrecedence()) || (o1.getPrecedence() < operators.get(nextToken.surface).getPrecedence()))) { outputQueue.add(stack.pop()); nextToken = stack.isEmpty() ? null : stack.peek(); } } public Value eval(Context c) { return eval(c, Context.NONE); } public Value eval(Context c, Integer expectedType) { if (ast == null) { ast = getAST(c); } return evalValue(() -> ast, c, expectedType); } public Value evalValue(Supplier exprProvider, Context c, Integer expectedType) { try { return exprProvider.get().evalValue(c, expectedType); } catch (ContinueStatement | BreakStatement | ReturnStatement exc) { throw new ExpressionException(c, this, "Control flow functions, like continue, break or return, should only be used in loops, and functions respectively."); } catch (ExitStatement exit) { return exit.retval==null?Value.NULL:exit.retval; } catch (StackOverflowError ignored) { throw new ExpressionException(c, this, "Your thoughts are too deep"); } catch (InternalExpressionException exc) { throw new ExpressionException(c, this, "Your expression result is incorrect: "+exc.getMessage()); } catch (ArithmeticException exc) { throw new ExpressionException(c, this, "The final result is incorrect: "+exc.getMessage()); } } private LazyValue getAST(Context context) { Stack stack = new Stack<>(); List rpn = shuntingYard(context); validate(context, rpn); for (final Tokenizer.Token token : rpn) { switch (token.type) { case UNARY_OPERATOR: { final LazyValue value = stack.pop(); LazyValue result = (c, t) -> operators.get(token.surface).lazyEval(c, t, this, token, value, null).evalValue(c, t); stack.push(result); break; } case OPERATOR: final LazyValue v1 = stack.pop(); final LazyValue v2 = stack.pop(); LazyValue result = (c,t) -> operators.get(token.surface).lazyEval(c, t,this, token, v2, v1).evalValue(c, t); stack.push(result); break; case VARIABLE: stack.push((c, t) -> getOrSetAnyVariable(c, token.surface).evalValue(c, t)); break; case FUNCTION: String name = token.surface; ILazyFunction f; ArrayList p; boolean isKnown = functions.containsKey(name); // globals will be evaluated lazily, not at compile time via . if (isKnown) { f = functions.get(name); p = new ArrayList<>(!f.numParamsVaries() ? f.getNumParams() : 0); } else // potentially unknown function or just unknown function { f = functions.get("call"); p = new ArrayList<>(); } // pop parameters off the stack until we hit the start of // this function's parameter list while (!stack.isEmpty() && stack.peek() != LazyValue.PARAMS_START) { p.add(stack.pop()); } if (!isKnown) p.add( (c, t) -> new StringValue(name)); Collections.reverse(p); if (stack.peek() == LazyValue.PARAMS_START) { stack.pop(); } stack.push((c, t) -> f.lazyEval(c, t, this, token, p).evalValue(c, t)); break; case OPEN_PAREN: stack.push(LazyValue.PARAMS_START); break; case LITERAL: stack.push((c, t) -> { try { return new NumericValue(token.surface); } catch (NumberFormatException exception) { throw new ExpressionException(c, this, token, "Not a number"); } }); break; case STRINGPARAM: stack.push((c, t) -> new StringValue(token.surface) ); // was originally null break; case HEX_LITERAL: stack.push((c, t) -> new NumericValue(new BigInteger(token.surface.substring(2), 16).doubleValue())); break; default: throw new ExpressionException(context, this, token, "Unexpected token '" + token.surface + "'"); } } return stack.pop(); } private void validate(Context c, List rpn) { /*- * Thanks to Norman Ramsey: * http://http://stackoverflow.com/questions/789847/postfix-notation-validation */ // each push on to this stack is a new function scope, with the value of // each // layer on the stack being the count of the number of parameters in // that scope Stack stack = new Stack<>(); // push the 'global' scope stack.push(0); for (final Tokenizer.Token token : rpn) { switch (token.type) { case UNARY_OPERATOR: if (stack.peek() < 1) { throw new ExpressionException(c, this, token, "Missing parameter(s) for operator " + token); } break; case OPERATOR: if (stack.peek() < 2) { if (token.surface.equals(";")) { throw new ExpressionException(c, this, token, "Empty expression found with ';'"); } throw new ExpressionException(c, this, token, "Missing parameter(s) for operator " + token); } // pop the operator's 2 parameters and add the result stack.set(stack.size() - 1, stack.peek() - 2 + 1); break; case FUNCTION: ILazyFunction f = functions.get(token.surface);// don't validate global - userdef functions int numParams = stack.pop(); if (f != null && !f.numParamsVaries() && numParams != f.getNumParams()) { throw new ExpressionException(c, this, token, "Function " + token + " expected " + f.getNumParams() + " parameters, got " + numParams); } if (stack.size() <= 0) { throw new ExpressionException(c, this, token, "Too many function calls, maximum scope exceeded"); } // push the result of the function stack.set(stack.size() - 1, stack.peek() + 1); break; case OPEN_PAREN: stack.push(0); break; default: stack.set(stack.size() - 1, stack.peek() + 1); } } if (stack.size() > 1) { throw new ExpressionException(c, this, "Too many unhandled function parameter lists"); } else if (stack.peek() > 1) { throw new ExpressionException(c, this, "Too many numbers or variables"); } else if (stack.peek() < 1) { throw new ExpressionException(c, this, "Empty expression"); } } }