diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/model/Token.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/model/Token.java index 1d69bad..567e722 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/model/Token.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/model/Token.java @@ -17,12 +17,12 @@ /** * A position the token starts */ - private Position from; + private Position start; /** * A position the token ends */ - private Position to; + private Position end; /** * Indicates whether the token is the minimal block or not @@ -37,8 +37,8 @@ */ public Token() { this.text = ""; - this.from = new Position(0, 0); - this.to = new Position(0, 0); + this.start = new Position(0, 0); + this.end = new Position(0, 0); this.atomic = false; } @@ -46,16 +46,16 @@ * Constructs a new token with given properties. * * @param text - * @param from - * @param to + * @param start + * @param end * @param atomic * @author Shohei Yamagiwa * @since 0.1 */ - public Token(final String text, final Position from, final Position to, final boolean atomic) { + public Token(final String text, final Position start, final Position end, final boolean atomic) { this.text = text; - this.from = from; - this.to = to; + this.start = start; + this.end = end; this.atomic = atomic; } @@ -67,20 +67,20 @@ this.text = text; } - public Position getPositionFrom() { - return from; + public Position getStartPos() { + return start; } - public void setPositionFrom(Position from) { - this.from = from; + public void setStartPos(Position start) { + this.start = start; } - public Position getPositionTo() { - return to; + public Position getEndPos() { + return end; } - public void setPositionTo(Position to) { - this.to = to; + public void setEndPos(Position end) { + this.end = end; } public boolean isAtomic() { @@ -93,7 +93,7 @@ @Override public String toString() { - return text + " " + "starts from " + "line: " + from.getLine() + ", " + "column: " + from.getColumn() + " " + "to" + " " + "line: " + to.getLine() + ", " + "column: " + to.getColumn() + ", " + "isAtomic: " + atomic; + return text + " " + "starts from " + "line: " + start.getLine() + ", " + "column: " + start.getColumn() + " " + "to" + " " + "line: " + end.getLine() + ", " + "column: " + end.getColumn() + ", " + "isAtomic: " + atomic; } @Override @@ -102,11 +102,11 @@ return false; } Token other = (Token) o; - return Objects.equals(text, other.text) && Objects.equals(from, other.from) && Objects.equals(to, other.to) && Objects.equals(atomic, other.atomic); + return Objects.equals(text, other.text) && Objects.equals(start, other.start) && Objects.equals(end, other.end) && Objects.equals(atomic, other.atomic); } @Override public int hashCode() { - return Objects.hash(text, from, to, atomic); + return Objects.hash(text, start, end, atomic); } } diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java index 9efaa8c..06ee260 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java @@ -7,7 +7,11 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Tokenizes the model file @@ -76,6 +80,8 @@ allTokens = splitBySpace(allTokens); allTokens = extractMultilineComments(allTokens); allTokens = extractComments(allTokens); + allTokens = splitBySymbol(allTokens, Tokens.LEFT_BRACKET, Tokens.LEFT_BRACKET_REGX); + allTokens = splitBySymbol(allTokens, Tokens.RIGHT_BRACKET, Tokens.RIGHT_BRACKET_REGX); return allTokens; } @@ -92,8 +98,8 @@ original.forEach(originalToken -> { String[] tokens = originalToken.getText().split(" ", -1); - int line = originalToken.getPositionFrom().getLine(); - int column = originalToken.getPositionFrom().getColumn(); + int line = originalToken.getStartPos().getLine(); + int column = originalToken.getStartPos().getColumn(); for (int i = 0; i < tokens.length; i++) { String prevToken = i != 0 ? tokens[i - 1] : null; String token = tokens[i]; @@ -140,7 +146,7 @@ switch (originalToken.getText()) { case Tokens.MULTILINE_COMMENT_START -> { commentToken = new Token(); - commentToken.setPositionFrom(originalToken.getPositionFrom()); + commentToken.setStartPos(originalToken.getStartPos()); commentText.append(originalToken.getText()); isComment = true; @@ -149,7 +155,7 @@ commentText.append(originalToken.getText()); isComment = false; - commentToken.setPositionTo(originalToken.getPositionTo()); + commentToken.setEndPos(originalToken.getEndPos()); commentToken.setText(commentText.toString()); commentToken.setAtomic(true); @@ -186,19 +192,19 @@ if (commentLine == -1) { commentToken = new Token(); commentToken.setText(originalToken.getText()); - commentToken.setPositionFrom(originalToken.getPositionFrom()); + commentToken.setStartPos(originalToken.getStartPos()); commentToken.setAtomic(true); - commentLine = originalToken.getPositionFrom().getLine(); - } else if (originalToken.getPositionFrom().getLine() == commentLine && originalToken.getPositionTo().getLine() == commentLine) { + commentLine = originalToken.getStartPos().getLine(); + } else if (originalToken.getStartPos().getLine() == commentLine && originalToken.getEndPos().getLine() == commentLine) { commentToken.setText(commentToken.getText() + originalToken.getText()); - commentToken.setPositionTo(originalToken.getPositionTo()); + commentToken.setEndPos(originalToken.getEndPos()); } } else { if (commentLine != -1) { - if (originalToken.getPositionFrom().getLine() == commentLine && originalToken.getPositionTo().getLine() == commentLine) { + if (originalToken.getStartPos().getLine() == commentLine && originalToken.getEndPos().getLine() == commentLine) { commentToken.setText(commentToken.getText() + originalToken.getText()); - commentToken.setPositionTo(originalToken.getPositionTo()); + commentToken.setEndPos(originalToken.getEndPos()); } else { newTokens.add(commentToken); newTokens.add(originalToken); @@ -211,4 +217,48 @@ } return newTokens; } + + private ArrayList splitBySymbol(final ArrayList original, final String symbolStr, final String symbolRegex) { + final int symbolLength = symbolStr.codePointCount(0, symbolStr.length()); + final Pattern pattern = Pattern.compile(symbolRegex); // Compile regex to use in the loop. + + final ArrayList newTokens = new ArrayList<>(); + + for (Token originalToken : original) { + if (originalToken.isAtomic()) { // Token is atomic so no more splits are needed + continue; + } + + /* Specified symbol isn't contained in the token */ + Matcher matcher = pattern.matcher(originalToken.getText()); + if (!matcher.find()) { + continue; + } + + // ( accounts ( accounts:List,signup(name:Str))) + boolean startsWithToken = originalToken.getText().startsWith(symbolStr); + boolean endsWithToken = originalToken.getText().endsWith(symbolStr); + ArrayDeque tokens = new ArrayDeque<>(Arrays.asList(originalToken.getText().split(symbolRegex))); + + /* Split into several tokens */ + ArrayList splitTokens = new ArrayList<>(); + Position start = originalToken.getStartPos(); + Position end = originalToken.getEndPos(); + + if (startsWithToken) { + Token symbol = new Token(symbolStr, start, start.move(0, symbolLength - 1), true); + + String restStr = tokens.poll(); + int restLength = restStr.codePointCount(0, restStr.length()); +// Token rest = new Token(restStr, , false); + } + for (String token : tokens) { + // TODO: Implement this line + } + if (endsWithToken) { + // TODO: Implement this line + } + } + return newTokens; + } }