diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java index dc7afbd..3e448bc 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java @@ -72,9 +72,11 @@ } /* Adds whole line as the token */ - allTokens.add(new Token(lineStr, new Position(line, column), new Position(line, column + lineStr.codePointCount(0, lineStr.length()) - 1), false)); + allTokens.add(new Token(lineStr, new Position(line, column), false)); line++; } + reader.close(); + allTokens = splitBySpace(allTokens); allTokens = extractMultilineComments(allTokens); allTokens = extractComments(allTokens); @@ -216,6 +218,18 @@ return newTokens; } + /** + * FIXME: Output string is not correct. + *
+ * Splits tokens into several new tokens with given symbol.
+ *
+ * @param original Original tokens
+ * @param symbolStr String of the symbol.
+ * @param symbolRegex Regex of the symbol
+ * @return New split {@link ArrayList} of {@link Token}
+ * @author Shohei Yamagiwa
+ * @since 0.1
+ */
protected static ArrayList