diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java index 2a241d2..106c45a 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java @@ -53,7 +53,57 @@ * @since 0.1 */ public final List execute() throws IOException { - List allTokens = new ArrayList<>(); + List tokens = splitByLine(reader); + tokens = splitBySpace(tokens); + tokens = extractMultilineComments(tokens); + tokens = extractComments(tokens); + + tokens = splitTokens(tokens, Tokens.EQ); + tokens = splitTokens(tokens, Tokens.NEQ); + tokens = splitTokens(tokens, Tokens.GE); + tokens = splitTokens(tokens, Tokens.LE); + tokens = splitTokens(tokens, Tokens.AND); + tokens = splitTokens(tokens, Tokens.OR); + + tokens = splitTokens(tokens, Tokens.LEFT_BRACKET); + tokens = splitTokens(tokens, Tokens.RIGHT_BRACKET); + tokens = splitTokens(tokens, Tokens.LEFT_CURLY_BRACKET); + tokens = splitTokens(tokens, Tokens.RIGHT_CURLY_BRACKET); + tokens = splitTokens(tokens, Tokens.LEFT_SQUARE_BRACKET); + tokens = splitTokens(tokens, Tokens.RIGHT_SQUARE_BRACKET); + + tokens = splitTokens(tokens, Tokens.ADD); + tokens = splitTokens(tokens, Tokens.SUB); + tokens = splitTokens(tokens, Tokens.MUL); + tokens = splitTokens(tokens, Tokens.DIV); + tokens = splitTokens(tokens, Tokens.MOD); + + tokens = splitTokens(tokens, Tokens.MINUS); + + tokens = splitTokens(tokens, Tokens.GT); + tokens = splitTokens(tokens, Tokens.LT); + tokens = splitTokens(tokens, Tokens.NEG); + + tokens = splitTokens(tokens, Tokens.EQUALS); + tokens = splitTokens(tokens, Tokens.ASSIGNMENT); + tokens = splitTokens(tokens, Tokens.COLON); + tokens = splitTokens(tokens, Tokens.COMMA); + tokens = splitTokens(tokens, Tokens.DOT); + tokens = splitTokens(tokens, Tokens.DOUBLE_QUOT); + return tokens; + } + + /** + * Reads lines from {@code reader} and convert each of them into a token. + * + * @param reader {@link BufferedReader} to read lines. + * @return {@link List} of converted tokens. + * @throws IOException Thrown when reading from buffer failed. + * @author Shohei Yamagiwa + * @since 0.1 + */ + protected static List splitByLine(BufferedReader reader) throws IOException { + List tokens = new ArrayList<>(); int line = 1; int column = 1; while (true) { @@ -71,48 +121,12 @@ } /* Adds whole line as the token */ - allTokens.add(new Token(lineStr, new Position(line, column), false)); + tokens.add(new Token(lineStr, new Position(line, column), false)); line++; } reader.close(); - allTokens = splitBySpace(allTokens); - allTokens = extractMultilineComments(allTokens); - allTokens = extractComments(allTokens); - - allTokens = splitTokens(allTokens, Tokens.EQ); - allTokens = splitTokens(allTokens, Tokens.NEQ); - allTokens = splitTokens(allTokens, Tokens.GE); - allTokens = splitTokens(allTokens, Tokens.LE); - allTokens = splitTokens(allTokens, Tokens.AND); - allTokens = splitTokens(allTokens, Tokens.OR); - - allTokens = splitTokens(allTokens, Tokens.LEFT_BRACKET); - allTokens = splitTokens(allTokens, Tokens.RIGHT_BRACKET); - allTokens = splitTokens(allTokens, Tokens.LEFT_CURLY_BRACKET); - allTokens = splitTokens(allTokens, Tokens.RIGHT_CURLY_BRACKET); - allTokens = splitTokens(allTokens, Tokens.LEFT_SQUARE_BRACKET); - allTokens = splitTokens(allTokens, Tokens.RIGHT_SQUARE_BRACKET); - - allTokens = splitTokens(allTokens, Tokens.ADD); - allTokens = splitTokens(allTokens, Tokens.SUB); - allTokens = splitTokens(allTokens, Tokens.MUL); - allTokens = splitTokens(allTokens, Tokens.DIV); - allTokens = splitTokens(allTokens, Tokens.MOD); - - allTokens = splitTokens(allTokens, Tokens.MINUS); - - allTokens = splitTokens(allTokens, Tokens.GT); - allTokens = splitTokens(allTokens, Tokens.LT); - allTokens = splitTokens(allTokens, Tokens.NEG); - - allTokens = splitTokens(allTokens, Tokens.EQUALS); - allTokens = splitTokens(allTokens, Tokens.ASSIGNMENT); - allTokens = splitTokens(allTokens, Tokens.COLON); - allTokens = splitTokens(allTokens, Tokens.COMMA); - allTokens = splitTokens(allTokens, Tokens.DOT); - allTokens = splitTokens(allTokens, Tokens.DOUBLE_QUOT); - return allTokens; + return tokens; } /**