diff --git a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java index 106c45a..9598c82 100644 --- a/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java +++ b/LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java @@ -18,41 +18,28 @@ */ public class Tokenizer { /** - * The {@link BufferedReader} that contains model text in memory. - */ - private final BufferedReader reader; - - /** - * Initializes {@link Tokenizer} with prepared {@link BufferedReader} + * Tokenizes the given text. * - * @param reader Prepared model text in {@link BufferedReader} - * @author Shohei Yamagiwa - * @since 0.1 - */ - public Tokenizer(BufferedReader reader) { - this.reader = reader; - } - - /** - * Initializes {@link Tokenizer} with given {@code text} - * - * @param text The text from specific model file. - * @author Shohei Yamagiwa - * @since 0.1 - */ - public Tokenizer(String text) { - this(new BufferedReader(new StringReader(text))); - } - - /** - * Tokenizes all texts in {@link BufferedReader} - * + * @param text The text to be tokenized. * @return All tokens in {@link List} * @throws IOException Be thrown if {@link BufferedReader} fails to read text lines * @author Shohei Yamagiwa * @since 0.1 */ - public final List execute() throws IOException { + public static List tokenize(String text) throws IOException { + return tokenize(new BufferedReader(new StringReader(text))); + } + + /** + * Tokenizes the text in given {@link BufferedReader} + * + * @param reader The text to be tokenized in the buffer + * @return All tokens in {@link List} + * @throws IOException Be thrown if {@link BufferedReader} fails to read text lines + * @author Shohei Yamagiwa + * @since 0.1 + */ + public static List tokenize(BufferedReader reader) throws IOException { List tokens = splitByLine(reader); tokens = splitBySpace(tokens); tokens = extractMultilineComments(tokens);