diff --git a/AlgebraicDataflowArchitectureModel/src/parser/Parser.java b/AlgebraicDataflowArchitectureModel/src/parser/Parser.java index b42a607..3697787 100644 --- a/AlgebraicDataflowArchitectureModel/src/parser/Parser.java +++ b/AlgebraicDataflowArchitectureModel/src/parser/Parser.java @@ -63,6 +63,10 @@ public static final String SINGLE_LINE_COMMENT = "//"; public static final String SINGLE_LINE_COMMENT_REGX = "\\/\\/"; + public static final String MULTILINE_COMMENT_START = "/*"; + public static final String MULTILINE_COMMENT_START_REGX = "\\/\\*"; + public static final String MULTILINE_COMMENT_END = "*/"; + public static final String MULTILINE_COMMENT_END_REGX = "\\*\\/"; public Parser(final TokenStream stream) { this.stream = stream; @@ -75,23 +79,18 @@ while ((line = reader.readLine()) != null) { stream.addLine(line); } + stream.removeMultilineComments(); reader.close(); } catch (IOException e) { e.printStackTrace(); } } - public DataTransferModel doParse() - throws ExpectedRightBracket, ExpectedChannel, ExpectedChannelName, ExpectedLeftCurlyBracket, ExpectedRightCurlyBracket, - ExpectedInOrOutOrRefOrSubKeyword, ExpectedStateTransition, ExpectedEquals, - ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, ExpectedAssignment, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public DataTransferModel doParse() throws ExpectedRightBracket, ExpectedChannel, ExpectedChannelName, ExpectedLeftCurlyBracket, ExpectedRightCurlyBracket, ExpectedInOrOutOrRefOrSubKeyword, ExpectedStateTransition, ExpectedEquals, ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, ExpectedAssignment, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { return parseDataFlowModel(); } - public DataTransferModel parseDataFlowModel() - throws ExpectedRightBracket, ExpectedChannel, ExpectedChannelName, ExpectedLeftCurlyBracket, ExpectedRightCurlyBracket, - ExpectedInOrOutOrRefOrSubKeyword, ExpectedStateTransition, ExpectedEquals, - ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, ExpectedAssignment, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public DataTransferModel parseDataFlowModel() throws ExpectedRightBracket, ExpectedChannel, ExpectedChannelName, ExpectedLeftCurlyBracket, ExpectedRightCurlyBracket, ExpectedInOrOutOrRefOrSubKeyword, ExpectedStateTransition, ExpectedEquals, ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, ExpectedAssignment, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { DataTransferModel model = new DataTransferModel(); DataTransferChannel channel; while ((channel = parseChannel(model)) != null) { @@ -104,11 +103,7 @@ return model; } - public DataTransferChannel parseChannel(DataTransferModel model) - throws ExpectedLeftCurlyBracket, ExpectedRightBracket, ExpectedRightCurlyBracket, ExpectedAssignment, - ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, - ExpectedChannel, ExpectedChannelName, ExpectedInOrOutOrRefOrSubKeyword, - ExpectedStateTransition, ExpectedEquals, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public DataTransferChannel parseChannel(DataTransferModel model) throws ExpectedLeftCurlyBracket, ExpectedRightBracket, ExpectedRightCurlyBracket, ExpectedAssignment, ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, ExpectedChannel, ExpectedChannelName, ExpectedInOrOutOrRefOrSubKeyword, ExpectedStateTransition, ExpectedEquals, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { if (!stream.hasNext()) return null; if (stream.checkNext().equals(RIGHT_CURLY_BRACKET)) return null; @@ -190,9 +185,7 @@ return channel; } - public void parseInit(DataTransferModel model) - throws ExpectedLeftCurlyBracket, ExpectedAssignment, ExpectedRHSExpression, WrongRHSExpression, - ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public void parseInit(DataTransferModel model) throws ExpectedLeftCurlyBracket, ExpectedAssignment, ExpectedRHSExpression, WrongRHSExpression, ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { String leftBracket = stream.next(); if (!leftBracket.equals(LEFT_CURLY_BRACKET)) throw new ExpectedLeftCurlyBracket(stream.getLine()); while (stream.hasNext() && !stream.checkNext().equals(RIGHT_CURLY_BRACKET)) { @@ -218,9 +211,7 @@ if (stream.hasNext()) stream.next(); } - public ChannelMember parseChannelMember(DataTransferModel model, final String inOrOutOrRef) - throws ExpectedRightBracket, ExpectedRightCurlyBracket, ExpectedStateTransition, ExpectedEquals, - ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public ChannelMember parseChannelMember(DataTransferModel model, final String inOrOutOrRef) throws ExpectedRightBracket, ExpectedRightCurlyBracket, ExpectedStateTransition, ExpectedEquals, ExpectedRHSExpression, WrongLHSExpression, WrongRHSExpression, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { if (!stream.hasNext()) throw new ExpectedStateTransition(stream.getLine()); StateTransitionTerm leftTerm = parseStateTransitionTerm(stream, model); if (leftTerm == null || !(leftTerm instanceof Term)) throw new WrongLHSExpression(stream.getLine()); @@ -507,8 +498,7 @@ term.addChild(first); term.addChild(second); first = term; - } else if (op.getName().equals(EQ) || op.getName().equals(NEQ) || op.getName().equals(GT) || op.getName().equals(LT) - || op.getName().equals(GE) || op.getName().equals(LE) || op.getName().equals(AND) || op.getName().equals(OR)) { + } else if (op.getName().equals(EQ) || op.getName().equals(NEQ) || op.getName().equals(GT) || op.getName().equals(LT) || op.getName().equals(GE) || op.getName().equals(LE) || op.getName().equals(AND) || op.getName().equals(OR)) { // lower priority than add and sub if (first != null) monomials.add(first); Expression firstMonomial = monomials.getFirst(); @@ -634,8 +624,7 @@ return var; } - public StateTransitionTerm parseStateTransitionTerm(TokenStream stream, DataTransferModel model) - throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public StateTransitionTerm parseStateTransitionTerm(TokenStream stream, DataTransferModel model) throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { ResourcePath resourcePath = parseResourcePath(stream, model); StateTransitionTerm term = new StateTransitionTerm(resourcePath); int arity = 0; @@ -652,8 +641,7 @@ return term; } - public ResourceHierarchy parseResourceHierarchy(TokenStream stream, DataTransferModel model) - throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression { + public ResourceHierarchy parseResourceHierarchy(TokenStream stream, DataTransferModel model) throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression { ResourceHierarchy hierarchy = null; do { String literalOrLeftCurlyBracket = stream.next(); @@ -675,8 +663,7 @@ return hierarchy; } - public ResourcePath parseResourcePath(TokenStream stream, DataTransferModel model) - throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { + public ResourcePath parseResourcePath(TokenStream stream, DataTransferModel model) throws ExpectedRightBracket, ExpectedRightCurlyBracket, WrongPathExpression, WrongJsonExpression, ExpectedColon, ExpectedDoubleQuotation { ResourcePath path = null; do { String literalOrLeftCurlyBracket = stream.next(); @@ -738,6 +725,8 @@ return; } ArrayList tokenList = splitByDoubleQuotation(line); + tokenList = splitBy(tokenList, MULTILINE_COMMENT_START, MULTILINE_COMMENT_START_REGX); + tokenList = splitBy(tokenList, MULTILINE_COMMENT_END, MULTILINE_COMMENT_END_REGX); tokenList = splitBy(tokenList, ADD, ADD_REGX); tokenList = splitBy(tokenList, MUL, MUL_REGX); tokenList = splitBy(tokenList, SUB, SUB_REGX); @@ -765,6 +754,32 @@ tokens.add(tokenList); } + public void removeMultilineComments() { + ArrayList> newTokens = new ArrayList<>(); + boolean isComment = false; + for (ArrayList tokenList : tokens) { + ArrayList newTokenList = new ArrayList<>(); + for (Token token : tokenList) { + if (token.getTokenStr().equals(MULTILINE_COMMENT_START)) { + isComment = true; + continue; + } + if (token.getTokenStr().equals(MULTILINE_COMMENT_END)) { + isComment = false; + continue; + } + if (isComment) { + continue; + } + newTokenList.add(token); + } + if (!newTokenList.isEmpty()) { + newTokens.add(newTokenList); + } + } + tokens = newTokens; + } + private ArrayList splitBy(final List tokens, final String delimiter, final String delimiterRegx) { ArrayList newTokens = new ArrayList<>(); for (Token token : tokens) { diff --git a/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineComment.model b/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineComment.model index ff4f99b..928cdf3 100644 --- a/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineComment.model +++ b/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineComment.model @@ -4,7 +4,12 @@ */ channel CreateAccount { /* Actually a single comment here but its okay */ - accounts(accounts: Map, createAccount(accountId: Int, username: Str, email: Str, password: Str)) = insert(accounts, accountId, {"username": username, "email": email, "password": password}); // A comment to describe this line + out accounts(accounts: Map, createAccount(accountId: Int, username: Str, email: Str, password: Str)) = insert(accounts, accountId, {"username": username, "email": email, "password": password}) // A comment to describe this line + /* + * Hello World + * こんにちは世界 + * 3rd comment here + */ } /* * 日本語で書かれたコメントです。 diff --git a/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineCommentTest.java b/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineCommentTest.java new file mode 100644 index 0000000..0f4c9b5 --- /dev/null +++ b/AlgebraicDataflowArchitectureModel/src/tests/parser/comment/MultilineCommentTest.java @@ -0,0 +1,105 @@ +package tests.parser.comment; + +import models.algebra.Symbol; +import models.algebra.Term; +import models.algebra.Variable; +import models.dataConstraintModel.ChannelMember; +import models.dataConstraintModel.JsonTerm; +import models.dataConstraintModel.ResourcePath; +import models.dataConstraintModel.StateTransition; +import models.dataFlowModel.DataTransferChannel; +import models.dataFlowModel.DataTransferModel; +import org.junit.Test; +import parser.Parser; +import parser.exceptions.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +import static org.junit.Assert.assertEquals; + +public class MultilineCommentTest { + @Test + public void test() { + try (InputStream in = getClass().getResourceAsStream("MultilineComment.model")) { + assert in != null; + Parser parser = new Parser(new BufferedReader(new InputStreamReader(in))); + try { + DataTransferModel actual = parser.doParse(); + DataTransferModel expected = getExpectedModel(); + // FIXME: Should assert by comparison of object members, not string equality + assertEquals(expected.toString(), actual.toString()); + } catch (ExpectedAssignment | WrongLHSExpression | ExpectedInOrOutOrRefOrSubKeyword | WrongJsonExpression | + WrongPathExpression | ExpectedRightCurlyBracket | ExpectedChannelName | ExpectedStateTransition | + ExpectedRightBracket | ExpectedRHSExpression | WrongRHSExpression | ExpectedEquals | + ExpectedLeftCurlyBracket | ExpectedColon | ExpectedDoubleQuotation | ExpectedChannel e) { + throw new RuntimeException(e); + } + } catch (RuntimeException | IOException e) { + throw new RuntimeException(e); + } + } + + private DataTransferModel getExpectedModel() { + ResourcePath accountsPath = new ResourcePath("accounts"); + ResourcePath accountPath = new ResourcePath(accountsPath, new Variable("accountId")); + ResourcePath usernamePath = new ResourcePath(accountPath, "username"); + ResourcePath emailPath = new ResourcePath(accountPath, "email"); + ResourcePath passwordPath = new ResourcePath(accountPath, "password"); + + DataTransferChannel createAccountChannel = new DataTransferChannel("CreateAccount"); + ChannelMember accountsChannelMember = new ChannelMember(accountsPath); + createAccountChannel.addChannelMemberAsOutput(accountsChannelMember); + + StateTransition transition = new StateTransition(); + transition.setCurStateExpression(getAccountsVariable()); + transition.setMessageExpression(getCreateAccountTerm()); + transition.setNextStateExpression(getInsertTerm()); + accountsChannelMember.setStateTransition(transition); + + DataTransferModel model = new DataTransferModel(); + model.addResourcePath(usernamePath); + model.addResourcePath(emailPath); + model.addResourcePath(passwordPath); + model.addInputChannel(createAccountChannel); + return model; + } + + private Variable getAccountsVariable() { + return new Variable("accounts", DataTransferModel.typeMap); + } + + private Term getCreateAccountTerm() { + Symbol createAccount = new Symbol("createAccount", 4); + Variable accountId = new Variable("accountId", DataTransferModel.typeInt); + Variable username = new Variable("username", DataTransferModel.typeString); + Variable email = new Variable("email", DataTransferModel.typeString); + Variable password = new Variable("password", DataTransferModel.typeString); + Term term = new Term(createAccount); + term.addChild(accountId); + term.addChild(username); + term.addChild(email); + term.addChild(password); + return term; + } + + private Term getInsertTerm() { + Variable accounts = new Variable("accounts"); + Variable accountId = new Variable("accountId"); + JsonTerm jsonBody = new JsonTerm(); + Variable username = new Variable("username"); + Variable email = new Variable("email"); + Variable password = new Variable("password"); + jsonBody.addMember("username", username); + jsonBody.addMember("email", email); + jsonBody.addMember("password", password); + Symbol insert = new Symbol("insert", 3); + Term term = new Term(insert); + term.addChild(accounts); + term.addChild(accountId); + term.addChild(jsonBody); + return term; + } +}