Tokenizer#splitBySymbol is still buggy.

Fork: 1

nitta-lab / AlgebraicDataflowArchitectureModel

Browse code Tokenizer#splitBySymbol is still buggy. language-server
1 parent 2bcf028 commit 86b56c0c9e5fb952dd848aa62cfb13370ba4d506 Shohei Yamagiwa authored 9 days ago

Patch

Unified Split

Showing 1 changed file

                Ignore Space
               Show notes
              View
            
          
          31 ■■■■■
          LanguageServer/src/main/java/org/nittalab/dtram/languageserver/utils/Tokenizer.java
-                continue;
            }
 
            /* Adds whole line as the token */
            allTokens.add(new Token(lineStr, new Position(line, column), new Position(line, column + lineStr.codePointCount(0, lineStr.length()) - 1), false));
            allTokens.add(new Token(lineStr, new Position(line, column), false));
            line++;
        }
        reader.close();
 
        allTokens = splitBySpace(allTokens);
        allTokens = extractMultilineComments(allTokens);
        allTokens = extractComments(allTokens);
        allTokens = splitBySymbol(allTokens, Tokens.LEFT_BRACKET, Tokens.LEFT_BRACKET_REGX);
        }
        return newTokens;
    }
 
    /**
     * FIXME: Output string is not correct.
     * <p>
     * Splits tokens into several new tokens with given symbol.
     *
     * @param original    Original tokens
     * @param symbolStr   String of the symbol.
     * @param symbolRegex Regex of the symbol
     * @return New split {@link ArrayList} of {@link Token}
     * @author Shohei Yamagiwa
     * @since 0.1
     */
    protected static ArrayList<Token> splitBySymbol(final ArrayList<Token> original, final String symbolStr, final String symbolRegex) {
        Pattern pattern = Pattern.compile(symbolRegex); // Compile regex to use in the loop.
        ArrayList<Token> newTokens = new ArrayList<>();
 
                }
            }
            newTokens.addAll(splitTokens);
        }
        System.out.println(constructTextFromTokens(newTokens));
        System.out.println();
        return newTokens;
    }
 
    private static String constructTextFromTokens(ArrayList<Token> tokens) {
        int lastLine = 1;
        StringBuilder builder = new StringBuilder();
        for (Token token : tokens) {
            if (token.getStartPos().getLine() != lastLine) {
                builder.append(System.lineSeparator());
            }
            builder.append(token.getText());
            lastLine = token.getEndPos().getLine();
        }
        return builder.toString();
    }
}
                 continue;
             }
             /* Adds whole line as the token */
+            allTokens.add(new Token(lineStr, new Position(line, column), new Position(line, column + lineStr.codePointCount(0, lineStr.length()) - 1), false));
+            allTokens.add(new Token(lineStr, new Position(line, column), false));
             line++;
         }
+        reader.close();
         allTokens = splitBySpace(allTokens);
         allTokens = extractMultilineComments(allTokens);
         allTokens = extractComments(allTokens);
         allTokens = splitBySymbol(allTokens, Tokens.LEFT_BRACKET, Tokens.LEFT_BRACKET_REGX);
         }
         return newTokens;
     }
+    /**
+     * FIXME: Output string is not correct.
+     * <p>
+     * Splits tokens into several new tokens with given symbol.
+     *
+     * @param original    Original tokens
+     * @param symbolStr   String of the symbol.
+     * @param symbolRegex Regex of the symbol
+     * @return New split {@link ArrayList} of {@link Token}
+     * @author Shohei Yamagiwa
+     * @since 0.1
+     */
     protected static ArrayList<Token> splitBySymbol(final ArrayList<Token> original, final String symbolStr, final String symbolRegex) {
         Pattern pattern = Pattern.compile(symbolRegex); // Compile regex to use in the loop.
         ArrayList<Token> newTokens = new ArrayList<>();
                 }
             }
             newTokens.addAll(splitTokens);
         }
+        System.out.println(constructTextFromTokens(newTokens));
+        System.out.println();
         return newTokens;
     }
+    private static String constructTextFromTokens(ArrayList<Token> tokens) {
+        int lastLine = 1;
+        StringBuilder builder = new StringBuilder();
+        for (Token token : tokens) {
+            if (token.getStartPos().getLine() != lastLine) {
+                builder.append(System.lineSeparator());
+            }
+            builder.append(token.getText());
+            lastLine = token.getEndPos().getLine();
+        }
+        return builder.toString();
+    }
 }

Show line notes below