From 6b3313b45ea03be92ee9f5595740799e6cc81ec8 Mon Sep 17 00:00:00 2001 From: gnembon Date: Wed, 3 Jul 2019 16:07:55 -0300 Subject: [PATCH 1/3] comments WIP --- src/main/java/carpet/script/Tokenizer.java | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/carpet/script/Tokenizer.java b/src/main/java/carpet/script/Tokenizer.java index f9196e21..bb97283b 100644 --- a/src/main/java/carpet/script/Tokenizer.java +++ b/src/main/java/carpet/script/Tokenizer.java @@ -22,6 +22,8 @@ public class Tokenizer implements Iterator private int pos = 0; private int lineno = 0; private int linepos = 0; + private boolean comments; + private boolean newLinesMarkers; /** The original input expression. */ @@ -31,10 +33,12 @@ public class Tokenizer implements Iterator private Expression expression; - Tokenizer(Expression expr, String input) + Tokenizer(Expression expr, String input, boolean allowComments, boolean allowNewLineMakers) { this.input = input; this.expression = expr; + this.comments = allowComments; + this.newLinesMarkers = allowNewLineMakers; } @@ -214,6 +218,22 @@ public class Tokenizer implements Iterator && (pos < input.length())) { greedyMatch += ch; + if (comments && "//".equals(greedyMatch)) + { + + while ( ch != '\n' && pos < input.length()) + { + ch = input.charAt(pos++); + linepos++; + greedyMatch += ch; + + } + if (ch=='\n') + { + lineno++; + linepos = 0; + } + } pos++; linepos++; if (Expression.none.isAnOperator(greedyMatch)) @@ -222,6 +242,14 @@ public class Tokenizer implements Iterator } ch = pos == input.length() ? 0 : input.charAt(pos); } + if (newLinesMarkers && "$".equals(greedyMatch)) + { + lineno++; + linepos = 0; + token.type = Token.TokenType.MARKER; + token.append('$'); + return token; // skipping previous token lookback + } if (validOperatorSeenUntil != -1) { token.append(input.substring(initialPos, validOperatorSeenUntil)); @@ -277,7 +305,7 @@ public class Tokenizer implements Iterator enum TokenType { VARIABLE, FUNCTION, LITERAL, OPERATOR, UNARY_OPERATOR, - OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM + OPEN_PAREN, COMMA, CLOSE_PAREN, HEX_LITERAL, STRINGPARAM, MARKER } public String surface = ""; public TokenType type; From b1df45a53b2f305611e4e905fe6014161481cbf2 Mon Sep 17 00:00:00 2001 From: gnembon Date: Wed, 3 Jul 2019 17:40:00 -0300 Subject: [PATCH 2/3] handling marker tokens --- gradle.properties | 4 +-- .../carpet/script/CarpetScriptServer.java | 2 ++ src/main/java/carpet/script/Expression.java | 29 +++++++++++++++---- src/main/java/carpet/script/ScriptHost.java | 1 + src/main/java/carpet/script/Tokenizer.java | 5 +++- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/gradle.properties b/gradle.properties index e73bb52d..815272f0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,8 +4,8 @@ org.gradle.jvmargs=-Xmx1G # Fabric Properties # check these on https://fabricmc.net/use # or better: https://modmuss50.me/fabric.html - minecraft_version=1.14.3 - yarn_mappings=1.14.3+build.1 + minecraft_version=1.14.4-pre1 + yarn_mappings=1.14.4-pre1+build.1 loader_version=0.4.8+build.155 # Mod Properties diff --git a/src/main/java/carpet/script/CarpetScriptServer.java b/src/main/java/carpet/script/CarpetScriptServer.java index d3bd038f..f812d518 100644 --- a/src/main/java/carpet/script/CarpetScriptServer.java +++ b/src/main/java/carpet/script/CarpetScriptServer.java @@ -146,6 +146,8 @@ public class CarpetScriptServer { setChatErrorSnooper(source); CarpetExpression ex = new CarpetExpression(module.getCode(), source, new BlockPos(0, 0, 0)); + ex.getExpr().asAModule(); + ex.scriptRunCommand(newHost, new BlockPos(source.getPosition())); } catch (CarpetExpressionException e) diff --git a/src/main/java/carpet/script/Expression.java b/src/main/java/carpet/script/Expression.java index 1410f39e..89badc14 100644 --- a/src/main/java/carpet/script/Expression.java +++ b/src/main/java/carpet/script/Expression.java @@ -335,6 +335,15 @@ public class Expression implements Cloneable private String expression; String getCodeString() {return expression;} + private boolean allowNewlineSubstitutions = true; + private boolean allowComments = false; + + void asAModule() + { + allowNewlineSubstitutions = false; + allowComments = true; + } + private String name; String getName() {return name;} Expression withName(String alias) @@ -2578,8 +2587,8 @@ public class Expression implements Cloneable */ public Expression(String expression) { - expression = expression.trim().replaceAll(";+$", ""); - this.expression = expression.replaceAll("\\$", "\n").trim(); + this.expression = expression.trim().replaceAll(";+$", ""); + //this.expression = expression.replaceAll("\\$", "\n").trim(); VariablesAndConstants(); UserDefinedFunctionsAndControlFlow(); Operators(); @@ -2588,12 +2597,12 @@ public class Expression implements Cloneable ListsLoopsAndHigherOrderFunctions(); } - private List shuntingYard(String expression) + private List shuntingYard() { List outputQueue = new ArrayList<>(); Stack stack = new Stack<>(); - Tokenizer tokenizer = new Tokenizer(this, expression); + Tokenizer tokenizer = new Tokenizer(this, expression, allowComments, allowNewlineSubstitutions); Tokenizer.Token lastFunction = null; Tokenizer.Token previousToken = null; @@ -2726,8 +2735,16 @@ public class Expression implements Cloneable { outputQueue.add(stack.pop()); } + case MARKER: + if ("$".equals(token.surface)) + { + StringBuilder sb = new StringBuilder(expression); + sb.setCharAt(token.pos, '\n'); + expression = sb.toString(); + } + break; } - previousToken = token; + if (token.type != Tokenizer.Token.TokenType.MARKER) previousToken = token; } while (!stack.isEmpty()) @@ -2796,7 +2813,7 @@ public class Expression implements Cloneable private LazyValue getAST() { Stack stack = new Stack<>(); - List rpn = shuntingYard(this.expression); + List rpn = shuntingYard(); validate(rpn); for (final Tokenizer.Token token : rpn) { diff --git a/src/main/java/carpet/script/ScriptHost.java b/src/main/java/carpet/script/ScriptHost.java index 4cede524..18852b7a 100644 --- a/src/main/java/carpet/script/ScriptHost.java +++ b/src/main/java/carpet/script/ScriptHost.java @@ -129,6 +129,7 @@ public class ScriptHost case OPEN_PAREN: case COMMA: case CLOSE_PAREN: + case MARKER: return "Fail: "+tok.surface+" is not allowed in invoke"; } } diff --git a/src/main/java/carpet/script/Tokenizer.java b/src/main/java/carpet/script/Tokenizer.java index bb97283b..e5f7ffbe 100644 --- a/src/main/java/carpet/script/Tokenizer.java +++ b/src/main/java/carpet/script/Tokenizer.java @@ -67,7 +67,7 @@ public class Tokenizer implements Iterator public static List simplepass(String input) { - Tokenizer tok = new Tokenizer(null, input); + Tokenizer tok = new Tokenizer(null, input, false, false); List res = new ArrayList<>(); while (tok.hasNext()) res.add(tok.next()); return res; @@ -233,6 +233,9 @@ public class Tokenizer implements Iterator lineno++; linepos = 0; } + token.append(greedyMatch); + token.type = Token.TokenType.MARKER; + return token; // skipping setting previous } pos++; linepos++; From 8d61c6a969635219c74d80fe14b863f84290c4c6 Mon Sep 17 00:00:00 2001 From: gnembon Date: Wed, 3 Jul 2019 21:51:42 -0300 Subject: [PATCH 3/3] better newline handling --- src/main/java/carpet/script/Expression.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/carpet/script/Expression.java b/src/main/java/carpet/script/Expression.java index 89badc14..f35a1c39 100644 --- a/src/main/java/carpet/script/Expression.java +++ b/src/main/java/carpet/script/Expression.java @@ -2587,7 +2587,9 @@ public class Expression implements Cloneable */ public Expression(String expression) { - this.expression = expression.trim().replaceAll(";+$", ""); + this.expression = expression.trim(). + replaceAll("\\r\\n?", "\n"). + replaceAll(";+$", ""); //this.expression = expression.replaceAll("\\$", "\n").trim(); VariablesAndConstants(); UserDefinedFunctionsAndControlFlow();