diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java index dc7afbd..3e448bc 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java @@ -72,9 +72,11 @@ } /* Adds whole line as the token */ - allTokens.add(new Token(lineStr, new Position(line, column), new Position(line, column + lineStr.codePointCount(0, lineStr.length()) - 1), false)); + allTokens.add(new Token(lineStr, new Position(line, column), false)); line++; } + reader.close(); + allTokens = splitBySpace(allTokens); allTokens = extractMultilineComments(allTokens); allTokens = extractComments(allTokens); @@ -216,6 +218,18 @@ return newTokens; } + /** + * FIXME: Output string is not correct. + *

+ * Splits tokens into several new tokens with given symbol. + * + * @param original Original tokens + * @param symbolStr String of the symbol. + * @param symbolRegex Regex of the symbol + * @return New split {@link ArrayList} of {@link Token} + * @author Shohei Yamagiwa + * @since 0.1 + */ protected static ArrayList splitBySymbol(final ArrayList original, final String symbolStr, final String symbolRegex) { Pattern pattern = Pattern.compile(symbolRegex); // Compile regex to use in the loop. ArrayList newTokens = new ArrayList<>(); @@ -278,6 +292,21 @@ } newTokens.addAll(splitTokens); } + System.out.println(constructTextFromTokens(newTokens)); + System.out.println(); return newTokens; } + + private static String constructTextFromTokens(ArrayList tokens) { + int lastLine = 1; + StringBuilder builder = new StringBuilder(); + for (Token token : tokens) { + if (token.getStartPos().getLine() != lastLine) { + builder.append(System.lineSeparator()); + } + builder.append(token.getText()); + lastLine = token.getEndPos().getLine(); + } + return builder.toString(); + } }