From 4975d34a88e8a06a246618119da96c13f1860612 Mon Sep 17 00:00:00 2001 From: samukce Date: Wed, 17 Jun 2020 23:49:40 -0700 Subject: [PATCH 01/11] Able to replace line by custom node by extension handler --- .../commonmark/internal/DocumentParser.java | 7 ++- .../internal/InlineParserContextImpl.java | 11 +++- .../commonmark/internal/InlineParserImpl.java | 60 ++++++++++++++++++- .../org/commonmark/parser/InlineParser.java | 20 +++++++ .../parser/InlineParserContext.java | 2 + .../java/org/commonmark/parser/Parser.java | 13 +++- .../java/org/commonmark/test/ParserTest.java | 18 ++++++ 7 files changed, 124 insertions(+), 7 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java index d1b43a7b4..bc208b214 100644 --- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java @@ -63,6 +63,7 @@ public class DocumentParser implements ParserState { private final List blockParserFactories; private final InlineParserFactory inlineParserFactory; private final List delimiterProcessors; + private final List nodeExtensions; private final DocumentBlockParser documentBlockParser; private final Map definitions = new LinkedHashMap<>(); @@ -71,10 +72,12 @@ public class DocumentParser implements ParserState { private Set allBlockParsers = new LinkedHashSet<>(); public DocumentParser(List blockParserFactories, InlineParserFactory inlineParserFactory, - List delimiterProcessors) { + List delimiterProcessors, + List nodeExtensions) { this.blockParserFactories = blockParserFactories; this.inlineParserFactory = inlineParserFactory; this.delimiterProcessors = delimiterProcessors; + this.nodeExtensions = nodeExtensions; this.documentBlockParser = new DocumentBlockParser(); activateBlockParser(this.documentBlockParser); @@ -415,7 +418,7 @@ private void addDefinitionsFrom(ParagraphParser paragraphParser) { * Walk through a block & children recursively, parsing string content into inline content where appropriate. */ private void processInlines() { - InlineParserContextImpl context = new InlineParserContextImpl(delimiterProcessors, definitions); + InlineParserContextImpl context = new InlineParserContextImpl(delimiterProcessors, definitions, nodeExtensions); InlineParser inlineParser = inlineParserFactory.create(context); for (BlockParser blockParser : allBlockParsers) { diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserContextImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserContextImpl.java index bff085ad8..ba84bbb59 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserContextImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserContextImpl.java @@ -1,6 +1,7 @@ package org.commonmark.internal; import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.parser.InlineParser; import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.delimiter.DelimiterProcessor; @@ -11,11 +12,14 @@ public class InlineParserContextImpl implements InlineParserContext { private final List delimiterProcessors; private final Map linkReferenceDefinitions; + private final List nodeExtensions; public InlineParserContextImpl(List delimiterProcessors, - Map linkReferenceDefinitions) { + Map linkReferenceDefinitions, + List nodeExtensions) { this.delimiterProcessors = delimiterProcessors; this.linkReferenceDefinitions = linkReferenceDefinitions; + this.nodeExtensions = nodeExtensions; } @Override @@ -27,4 +31,9 @@ public List getCustomDelimiterProcessors() { public LinkReferenceDefinition getLinkReferenceDefinition(String label) { return linkReferenceDefinitions.get(label); } + + @Override + public List nodeExtensions() { + return nodeExtensions; + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index dfb50149a..ff20d8919 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -6,12 +6,26 @@ import org.commonmark.internal.util.Html5Entities; import org.commonmark.internal.util.LinkScanner; import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; +import org.commonmark.node.Code; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.Link; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.Node; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.Text; import org.commonmark.parser.InlineParser; import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.delimiter.DelimiterProcessor; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -56,6 +70,7 @@ public class InlineParserImpl implements InlineParser { private final BitSet delimiterCharacters; private final Map delimiterProcessors; private final InlineParserContext context; + private final List nodeExtensions; private String input; private int index; @@ -77,6 +92,7 @@ public InlineParserImpl(InlineParserContext inlineParserContext) { this.specialCharacters = calculateSpecialCharacters(delimiterCharacters); this.context = inlineParserContext; + this.nodeExtensions = inlineParserContext.nodeExtensions(); } public static BitSet calculateDelimiterCharacters(Set characters) { @@ -148,6 +164,11 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr public void parse(String content, Node block) { reset(content.trim()); + List nodes = customNodesByExtensions(); + for (Node node : nodes) { + block.appendChild(node); + } + Node previous = null; while (true) { Node node = parseInline(previous); @@ -384,6 +405,41 @@ private Node parseDelimiters(DelimiterProcessor delimiterProcessor, char delimit return node; } + /** + * Attempt to parse delimiters like emphasis, strong emphasis or custom delimiters. + */ + private List customNodesByExtensions() { +// DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar); +// if (res == null) { +// return null; +// } +// int length = res.count; + int startIndex = index; +// +// index += length; + List nodes = new ArrayList<>(); + + for (NodeExtension nodeExtension : nodeExtensions) { + List inlineBreakdowns = nodeExtension.lookup(input); + + for (NodeExtension.InlineBreakdown breakdown : inlineBreakdowns) { + nodes.add(breakdown.getNode()); + } + } + +// Text node = text(input); + + // Add entry to stack for this opener +// lastDelimiter = new Delimiter(node, delimiterChar, res.canOpen, res.canClose, lastDelimiter); +// lastDelimiter.length = length; +// lastDelimiter.originalLength = length; +// if (lastDelimiter.previous != null) { +// lastDelimiter.previous.next = lastDelimiter; +// } + + return nodes; + } + /** * Add open bracket to delimiter stack and add a text node to block's children. */ diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java index 492c3cc8a..9b47e3728 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java @@ -2,6 +2,8 @@ import org.commonmark.node.Node; +import java.util.List; + /** * Parser for inline content (text, links, emphasized text, etc). */ @@ -12,4 +14,22 @@ public interface InlineParser { * @param node the node to append resulting nodes to (as children) */ void parse(String input, Node node); + + interface NodeExtension { + List lookup(String inline); + + class InlineBreakdown { + int startIndex; + int endIndex; + Node node; + + public InlineBreakdown(Node node) { + this.node = node; + } + + public Node getNode() { + return this.node; + } + } + } } diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParserContext.java b/commonmark/src/main/java/org/commonmark/parser/InlineParserContext.java index 467742e2c..c97c48aa8 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParserContext.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParserContext.java @@ -22,4 +22,6 @@ public interface InlineParserContext { * @return the definition if one exists, {@code null} otherwise */ LinkReferenceDefinition getLinkReferenceDefinition(String label); + + List nodeExtensions(); } diff --git a/commonmark/src/main/java/org/commonmark/parser/Parser.java b/commonmark/src/main/java/org/commonmark/parser/Parser.java index 5e15158ad..ba1351210 100644 --- a/commonmark/src/main/java/org/commonmark/parser/Parser.java +++ b/commonmark/src/main/java/org/commonmark/parser/Parser.java @@ -31,17 +31,20 @@ public class Parser { private final List delimiterProcessors; private final InlineParserFactory inlineParserFactory; private final List postProcessors; + private final List nodeExtensions; private Parser(Builder builder) { this.blockParserFactories = DocumentParser.calculateBlockParserFactories(builder.blockParserFactories, builder.enabledBlockTypes); this.inlineParserFactory = builder.getInlineParserFactory(); this.postProcessors = builder.postProcessors; this.delimiterProcessors = builder.delimiterProcessors; + this.nodeExtensions = builder.nodeExtensions; // Try to construct an inline parser. Invalid configuration might result in an exception, which we want to // detect as soon as possible. this.inlineParserFactory.create(new InlineParserContextImpl(delimiterProcessors, - Collections.emptyMap())); + Collections.emptyMap(), + nodeExtensions)); } /** @@ -99,7 +102,7 @@ public Node parseReader(Reader input) throws IOException { } private DocumentParser createDocumentParser() { - return new DocumentParser(blockParserFactories, inlineParserFactory, delimiterProcessors); + return new DocumentParser(blockParserFactories, inlineParserFactory, delimiterProcessors, nodeExtensions); } private Node postProcess(Node document) { @@ -115,6 +118,7 @@ private Node postProcess(Node document) { public static class Builder { private final List blockParserFactories = new ArrayList<>(); private final List delimiterProcessors = new ArrayList<>(); + private final List nodeExtensions = new ArrayList<>(); private final List postProcessors = new ArrayList<>(); private Set> enabledBlockTypes = DocumentParser.getDefaultBlockParserTypes(); private InlineParserFactory inlineParserFactory; @@ -257,6 +261,11 @@ public InlineParser create(InlineParserContext inlineParserContext) { } }; } + + public Builder nodeExtension(InlineParser.NodeExtension nodeExtension) { + this.nodeExtensions.add(nodeExtension); + return this; + } } /** diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index e058de378..041fb534e 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -15,6 +15,7 @@ import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -23,6 +24,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -122,6 +125,21 @@ public InlineParser create(InlineParserContext inlineParserContext) { assertThat(parser.parse(input).getFirstChild().getFirstChild(), instanceOf(ThematicBreak.class)); } + @Test + public void convertLineToCustomNodeByNodeExtensionHandler() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(new InlineBreakdown(new Image())); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + String input = "some text to be converted to image node by node extension handler"; + + Node document = parser.parse(input); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); + } + @Test public void threading() throws Exception { final Parser parser = Parser.builder().build(); From c9db9437db3efcb513b7fb42f82b33e9d97b685b Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 01:31:13 -0700 Subject: [PATCH 02/11] Able to support mixed line with custom node and text --- .../commonmark/internal/InlineParserImpl.java | 73 +++++++++---------- .../org/commonmark/parser/InlineParser.java | 18 ++++- .../java/org/commonmark/test/ParserTest.java | 51 ++++++++++++- 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index ff20d8919..f45e40284 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -19,6 +19,7 @@ import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.delimiter.DelimiterProcessor; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -164,14 +165,22 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr public void parse(String content, Node block) { reset(content.trim()); - List nodes = customNodesByExtensions(); - for (Node node : nodes) { - block.appendChild(node); + boolean[] filledIndex = new boolean[input.length()]; + HashMap nodeExtensionsByBeginIndex = new HashMap<>(); + ArrayDeque customNodes = new ArrayDeque<>(); + + for (NodeExtension.InlineBreakdown inlineBreakdownNode : customNodesByExtensions()) { + customNodes.push(inlineBreakdownNode); + nodeExtensionsByBeginIndex.put(inlineBreakdownNode.getBeginIndex(), inlineBreakdownNode.getNode()); + + for (int i = inlineBreakdownNode.getBeginIndex(); i <= inlineBreakdownNode.getEndIndex(); i++) { + filledIndex[i] = true; + } } Node previous = null; while (true) { - Node node = parseInline(previous); + Node node = parseInline(previous, filledIndex, nodeExtensionsByBeginIndex, customNodes); previous = node; if (node != null) { block.appendChild(node); @@ -205,13 +214,28 @@ private Text text(String text) { * On success, return the new inline node. * On failure, return null. */ - private Node parseInline(Node previous) { + private Node parseInline(Node previous, boolean[] filledIndex, Map nodeExtensionsByBeginIndex, + ArrayDeque customNodes) { + Node node = nodeExtensionsByBeginIndex.get(index); + if (node != null) { + customNodes.pollFirst(); + index++; + return node; + } + + while (index < filledIndex.length && filledIndex[index]) { + index++; + } + + int nextBeginIndexForCustomNode = customNodes.isEmpty() + ? input.length() + : customNodes.getFirst().getBeginIndex(); + char c = peek(); if (c == '\0') { return null; } - Node node; switch (c) { case '\n': node = parseNewline(previous); @@ -246,7 +270,7 @@ private Node parseInline(Node previous) { DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c); node = parseDelimiters(delimiterProcessor, c); } else { - node = parseString(); + node = parseString(Math.min(nextBeginIndexForCustomNode, input.length())); } break; } @@ -405,38 +429,13 @@ private Node parseDelimiters(DelimiterProcessor delimiterProcessor, char delimit return node; } - /** - * Attempt to parse delimiters like emphasis, strong emphasis or custom delimiters. - */ - private List customNodesByExtensions() { -// DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar); -// if (res == null) { -// return null; -// } -// int length = res.count; - int startIndex = index; -// -// index += length; - List nodes = new ArrayList<>(); + private List customNodesByExtensions() { + List nodes = new ArrayList<>(); for (NodeExtension nodeExtension : nodeExtensions) { - List inlineBreakdowns = nodeExtension.lookup(input); - - for (NodeExtension.InlineBreakdown breakdown : inlineBreakdowns) { - nodes.add(breakdown.getNode()); - } + nodes.addAll(nodeExtension.lookup(input)); } -// Text node = text(input); - - // Add entry to stack for this opener -// lastDelimiter = new Delimiter(node, delimiterChar, res.canOpen, res.canClose, lastDelimiter); -// lastDelimiter.length = length; -// lastDelimiter.originalLength = length; -// if (lastDelimiter.previous != null) { -// lastDelimiter.previous.next = lastDelimiter; -// } - return nodes; } @@ -708,10 +707,10 @@ private Node parseEntity() { /** * Parse a run of ordinary characters, or a single character with a special meaning in markdown, as a plain string. + * @param length */ - private Node parseString() { + private Node parseString(int length) { int begin = index; - int length = input.length(); while (index != length) { if (specialCharacters.get(input.charAt(index))) { break; diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java index 9b47e3728..fb22d455a 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java @@ -19,17 +19,27 @@ interface NodeExtension { List lookup(String inline); class InlineBreakdown { - int startIndex; - int endIndex; - Node node; + private final Node node; + private final int beginIndex; + private final int endIndex; - public InlineBreakdown(Node node) { + public InlineBreakdown(Node node, int beginIndex, int endIndex) { this.node = node; + this.beginIndex = beginIndex; + this.endIndex = endIndex; } public Node getNode() { return this.node; } + + public int getBeginIndex() { + return beginIndex; + } + + public int getEndIndex() { + return endIndex; + } } } } diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index 041fb534e..2beb6cacf 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -15,7 +15,6 @@ import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -24,7 +23,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.assertEquals; @@ -130,7 +128,7 @@ public void convertLineToCustomNodeByNodeExtensionHandler() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(new InlineBreakdown(new Image())); + return singletonList(new InlineBreakdown(new Image(), 0, 64)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -140,6 +138,53 @@ public List lookup(String inline) { assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); } + @Test + public void convertLineByNodeExtensionWithCustomNodeInTheLineBegin() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(new InlineBreakdown(new Image(), 0, 17)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + String input = "CONVERTED_TO_IMAGE some"; + + Node document = parser.parse(input); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); + assertThat(document.getFirstChild().getLastChild(), instanceOf(Text.class)); + } + + @Test + public void convertLineByNodeExtensionWithCustomNodeInTheLineEnd() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(new InlineBreakdown(new Image(), 5, 22)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + String input = "some CONVERTED_TO_IMAGE"; + + Node document = parser.parse(input); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(document.getFirstChild().getLastChild(), instanceOf(Image.class)); + } + + // test custom in the middle of text // some CONVERTED_TO_IMAGE foo NEW_TO_IMAGE too + + // test more then one in the same line + + // test wrong begin/end index by the extension parser + + // guarantee sort List by beginIndex + + // test more then one extension registered + + // test to guarantee customExtension priority over DelimiterProcessor + + + + @Test public void threading() throws Exception { final Parser parser = Parser.builder().build(); From 4ddb4f54c6e1908594b5a0b0aff9b789231bdd2a Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 01:41:46 -0700 Subject: [PATCH 03/11] Simplify unit tests --- .../java/org/commonmark/test/ParserTest.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index 2beb6cacf..aeb45a22d 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -128,13 +128,12 @@ public void convertLineToCustomNodeByNodeExtensionHandler() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(new InlineBreakdown(new Image(), 0, 64)); + return singletonList(new InlineBreakdown(new Image(), 0, 8)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); - String input = "some text to be converted to image node by node extension handler"; - Node document = parser.parse(input); + Node document = parser.parse("some text"); assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); } @@ -147,11 +146,11 @@ public List lookup(String inline) { } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); - String input = "CONVERTED_TO_IMAGE some"; - Node document = parser.parse(input); + Node document = parser.parse("CONVERTED_TO_IMAGE some"); assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); assertThat(document.getFirstChild().getLastChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getLastChild()), equalTo(" some")); } @Test @@ -163,10 +162,10 @@ public List lookup(String inline) { } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); - String input = "some CONVERTED_TO_IMAGE"; - Node document = parser.parse(input); + Node document = parser.parse("some CONVERTED_TO_IMAGE"); assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); assertThat(document.getFirstChild().getLastChild(), instanceOf(Image.class)); } @@ -183,6 +182,8 @@ public List lookup(String inline) { // test to guarantee customExtension priority over DelimiterProcessor + // benchmark with custom vs without custom + @Test @@ -217,7 +218,14 @@ private String firstText(Node n) { assertThat(n, notNullValue()); n = n.getFirstChild(); } - return ((Text) n).getLiteral(); + return getLiteral(n); + } + + private String getLiteral(Node node) { + if (node instanceof Text) { + return ((Text)node).getLiteral(); + } + return node.toString(); } private static class DashBlock extends CustomBlock { From 708e0b387d0c2c116c97d35cc34f90624fdee2ec Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 22:00:47 -0700 Subject: [PATCH 04/11] Custom nodes by line mixed with texts --- .../commonmark/internal/InlineParserImpl.java | 50 ++--- .../org/commonmark/parser/InlineParser.java | 16 +- .../java/org/commonmark/test/ParserTest.java | 173 ++++++++++++++++-- 3 files changed, 198 insertions(+), 41 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index f45e40284..e6d7d9d89 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -165,22 +167,15 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr public void parse(String content, Node block) { reset(content.trim()); - boolean[] filledIndex = new boolean[input.length()]; - HashMap nodeExtensionsByBeginIndex = new HashMap<>(); ArrayDeque customNodes = new ArrayDeque<>(); for (NodeExtension.InlineBreakdown inlineBreakdownNode : customNodesByExtensions()) { - customNodes.push(inlineBreakdownNode); - nodeExtensionsByBeginIndex.put(inlineBreakdownNode.getBeginIndex(), inlineBreakdownNode.getNode()); - - for (int i = inlineBreakdownNode.getBeginIndex(); i <= inlineBreakdownNode.getEndIndex(); i++) { - filledIndex[i] = true; - } + customNodes.add(inlineBreakdownNode); } Node previous = null; while (true) { - Node node = parseInline(previous, filledIndex, nodeExtensionsByBeginIndex, customNodes); + Node node = parseInline(previous, customNodes); previous = node; if (node != null) { block.appendChild(node); @@ -214,28 +209,26 @@ private Text text(String text) { * On success, return the new inline node. * On failure, return null. */ - private Node parseInline(Node previous, boolean[] filledIndex, Map nodeExtensionsByBeginIndex, - ArrayDeque customNodes) { - Node node = nodeExtensionsByBeginIndex.get(index); - if (node != null) { - customNodes.pollFirst(); - index++; - return node; - } - - while (index < filledIndex.length && filledIndex[index]) { - index++; + private Node parseInline(Node previous, ArrayDeque customNodes) { + NodeExtension.InlineBreakdown inlineBreakdown = customNodes.isEmpty() + ? NodeExtension.InlineBreakdown.EMPTY + : customNodes.getFirst(); + + if (inlineBreakdown.getBeginIndex() == index) { + while (index <= inlineBreakdown.getEndIndex()) { + index++; + } + return customNodes.pollFirst().getNode(); } - int nextBeginIndexForCustomNode = customNodes.isEmpty() - ? input.length() - : customNodes.getFirst().getBeginIndex(); + int nextBeginIndexForCustomNode = inlineBreakdown.getBeginIndex(); char c = peek(); if (c == '\0') { return null; } + Node node; switch (c) { case '\n': node = parseNewline(previous); @@ -433,9 +426,18 @@ private List customNodesByExtensions() { List nodes = new ArrayList<>(); for (NodeExtension nodeExtension : nodeExtensions) { - nodes.addAll(nodeExtension.lookup(input)); + List inlineBreakdowns = nodeExtension.lookup(input); + if (inlineBreakdowns != null) { + nodes.addAll(inlineBreakdowns); + } } + Collections.sort(nodes, new Comparator() { + @Override + public int compare(NodeExtension.InlineBreakdown node1, NodeExtension.InlineBreakdown node2) { + return Integer.compare(node1.getBeginIndex(), node2.getBeginIndex()); + } + }); return nodes; } diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java index fb22d455a..175ddc455 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java @@ -19,11 +19,13 @@ interface NodeExtension { List lookup(String inline); class InlineBreakdown { + public static final InlineBreakdown EMPTY = + new InlineBreakdown(null, Integer.MAX_VALUE, Integer.MAX_VALUE); private final Node node; private final int beginIndex; private final int endIndex; - public InlineBreakdown(Node node, int beginIndex, int endIndex) { + private InlineBreakdown(Node node, int beginIndex, int endIndex) { this.node = node; this.beginIndex = beginIndex; this.endIndex = endIndex; @@ -40,6 +42,18 @@ public int getBeginIndex() { public int getEndIndex() { return endIndex; } + + public static InlineBreakdown of(Node node, int beginIndex, int endIndex) { + if (endIndex < beginIndex){ + throw new IllegalArgumentException("beginIndex "+ beginIndex +" must be greater then endIndex " + endIndex); + } + + if (beginIndex == endIndex) { + return EMPTY; + } + + return new InlineBreakdown(node, beginIndex, endIndex); + } } } } diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index aeb45a22d..635f13685 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -1,11 +1,24 @@ package org.commonmark.test; -import org.commonmark.node.*; +import org.commonmark.node.Block; +import org.commonmark.node.BulletList; +import org.commonmark.node.CustomBlock; +import org.commonmark.node.Heading; +import org.commonmark.node.Image; +import org.commonmark.node.Node; +import org.commonmark.node.Paragraph; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; import org.commonmark.parser.InlineParser; import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.InlineParserFactory; import org.commonmark.parser.Parser; -import org.commonmark.parser.block.*; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; import org.commonmark.renderer.html.HtmlRenderer; import org.commonmark.testutil.TestResources; import org.junit.Test; @@ -13,7 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -23,8 +36,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -36,7 +54,7 @@ public void ioReaderTest() throws IOException { InputStream input1 = TestResources.getSpec().openStream(); Node document1; - try (InputStreamReader reader = new InputStreamReader(input1, Charset.forName("UTF-8"))) { + try (InputStreamReader reader = new InputStreamReader(input1, StandardCharsets.UTF_8)) { document1 = parser.parseReader(reader); } @@ -128,7 +146,7 @@ public void convertLineToCustomNodeByNodeExtensionHandler() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(new InlineBreakdown(new Image(), 0, 8)); + return singletonList(InlineBreakdown.of(new Image(), 0, 8)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -137,12 +155,57 @@ public List lookup(String inline) { assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); } + @Test + public void convertLineByNodeExtensionLeftDefaultTextIfEmptyListFromExtension() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return emptyList(); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + + Node document = parser.parse("some text"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some text")); + } + + @Test + public void convertLineByNodeExtensionLeftDefaultTextIfNulListFromExtension() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return null; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + + Node document = parser.parse("some text"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some text")); + } + + @Test + public void convertLineByNodeExtensionWithCustomNodeInTheLineBeginOneSingleCharecter() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image(), 0, 1)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + + Node document = parser.parse("C"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Image.class)); + assertThat(document.getFirstChild().getLastChild(), instanceOf(Image.class)); + } + @Test public void convertLineByNodeExtensionWithCustomNodeInTheLineBegin() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(new InlineBreakdown(new Image(), 0, 17)); + return singletonList(InlineBreakdown.of(new Image(), 0, 17)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -158,7 +221,7 @@ public void convertLineByNodeExtensionWithCustomNodeInTheLineEnd() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(new InlineBreakdown(new Image(), 5, 22)); + return singletonList(InlineBreakdown.of(new Image(), 5, 22)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -169,22 +232,100 @@ public List lookup(String inline) { assertThat(document.getFirstChild().getLastChild(), instanceOf(Image.class)); } - // test custom in the middle of text // some CONVERTED_TO_IMAGE foo NEW_TO_IMAGE too - - // test more then one in the same line + @Test + public void convertLineByNodeExtensionWithCustomNodeInTheMiddle() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image(), 5, 22)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); - // test wrong begin/end index by the extension parser + Node document = parser.parse("some CONVERTED_TO_IMAGE second"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); + assertThat(document.getFirstChild().getFirstChild().getNext(), instanceOf(Image.class)); + assertThat(document.getFirstChild().getLastChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getLastChild()), equalTo(" second")); + } - // guarantee sort List by beginIndex + @Test + public void convertLineByNodeExtensionWithMoreThenOneOccurrence() { + final Node image1 = new Image(); + final Node image2 = new Image(); + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + List inlinesBreakdown = new ArrayList<>(); + inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 24)); + inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 52)); + return inlinesBreakdown; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); - // test more then one extension registered + Node document = parser.parse("some CONVERTED_TO_IMAGE_1 second CONVERTED_TO_IMAGE_2 third"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); + assertThat(document.getFirstChild().getFirstChild().getNext(), equalTo(image1)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext()), + equalTo(" second ")); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext(), equalTo(image2)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext(), + instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext()), + equalTo(" third")); + } - // test to guarantee customExtension priority over DelimiterProcessor + @Test + public void convertLineByNodeExtensionWithMoreThenOneOccurrenceSortingByIndex() { + final Node image1 = new Image(); + final Node image2 = new Image(); + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + List inlinesBreakdown = new ArrayList<>(); + inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 52)); + inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 24)); + return inlinesBreakdown; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + Node document = parser.parse("some CONVERTED_TO_IMAGE_1 second CONVERTED_TO_IMAGE_2 third"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); + assertThat(document.getFirstChild().getFirstChild().getNext(), equalTo(image1)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext()), + equalTo(" second ")); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext(), equalTo(image2)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext(), + instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext()), + equalTo(" third")); + } - // benchmark with custom vs without custom + @Test + public void convertLineByNodeExtensionShouldIgnoreIfBeginIndexEqualEndIndex() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image(), 5, 5)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + Node document = parser.parse("some second"); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some second")); + } + @Test(expected = IllegalArgumentException.class) + public void convertLineByNodeExtensionShouldRaiseExceptionIfBeginIndexLessThenEndIndex() { + InlineParser.NodeExtension.InlineBreakdown.of(new Image(), 4, 3); + } @Test public void threading() throws Exception { From e364a0e7ee03c718c86eb766e02a7a73e2573243 Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 22:36:50 -0700 Subject: [PATCH 05/11] Fix index to take next custom node --- .../commonmark/internal/InlineParserImpl.java | 8 +-- .../java/org/commonmark/test/ParserTest.java | 49 ++++++++++++++----- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index e6d7d9d89..841bbdc55 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -213,16 +213,18 @@ private Node parseInline(Node previous, ArrayDeque lookup(String inline) { } @Test - public void convertLineByNodeExtensionWithCustomNodeInTheLineBeginOneSingleCharecter() { + public void convertLineByNodeExtensionWithCustomNodeInTheLineBeginOneSingleCharacter() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { @@ -205,7 +205,7 @@ public void convertLineByNodeExtensionWithCustomNodeInTheLineBegin() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(InlineBreakdown.of(new Image(), 0, 17)); + return singletonList(InlineBreakdown.of(new Image(), 0, 18)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -221,7 +221,7 @@ public void convertLineByNodeExtensionWithCustomNodeInTheLineEnd() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(InlineBreakdown.of(new Image(), 5, 22)); + return singletonList(InlineBreakdown.of(new Image(), 5, 23)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -229,7 +229,7 @@ public List lookup(String inline) { Node document = parser.parse("some CONVERTED_TO_IMAGE"); assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); - assertThat(document.getFirstChild().getLastChild(), instanceOf(Image.class)); + assertThat(document.getFirstChild().getFirstChild().getNext(), instanceOf(Image.class)); } @Test @@ -237,7 +237,7 @@ public void convertLineByNodeExtensionWithCustomNodeInTheMiddle() { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { @Override public List lookup(String inline) { - return singletonList(InlineBreakdown.of(new Image(), 5, 22)); + return singletonList(InlineBreakdown.of(new Image(), 5, 23)); } }; Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); @@ -258,8 +258,8 @@ public void convertLineByNodeExtensionWithMoreThenOneOccurrence() { @Override public List lookup(String inline) { List inlinesBreakdown = new ArrayList<>(); - inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 24)); - inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 52)); + inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 25)); + inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 53)); return inlinesBreakdown; } }; @@ -287,8 +287,8 @@ public void convertLineByNodeExtensionWithMoreThenOneOccurrenceSortingByIndex() @Override public List lookup(String inline) { List inlinesBreakdown = new ArrayList<>(); - inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 52)); - inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 24)); + inlinesBreakdown.add(InlineBreakdown.of(image2, 33, 53)); + inlinesBreakdown.add(InlineBreakdown.of(image1, 5, 25)); return inlinesBreakdown; } }; @@ -327,9 +327,36 @@ public void convertLineByNodeExtensionShouldRaiseExceptionIfBeginIndexLessThenEn InlineParser.NodeExtension.InlineBreakdown.of(new Image(), 4, 3); } + @Test + public void convertLineByNodeExtensionShouldIgnoreTheNextComponentIfStartingBeforeThePreviousEndIndex() { + final Node image1 = new Image(); + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + List inlinesBreakdown = new ArrayList<>(); + inlinesBreakdown.add(InlineBreakdown.of(image1, 0, 3)); + inlinesBreakdown.add(InlineBreakdown.of(new Image(), 2, 5)); + return inlinesBreakdown; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + + Node document = parser.parse("foo jack some"); + assertThat(document.getFirstChild().getFirstChild(), equalTo(image1)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext()), + equalTo(" jack some")); + } + @Test public void threading() throws Exception { - final Parser parser = Parser.builder().build(); + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image("www.image", "some"), 5, 6)); + } + }; + + final Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); final String spec = TestResources.readAsString(TestResources.getSpec()); HtmlRenderer renderer = HtmlRenderer.builder().build(); @@ -341,7 +368,7 @@ public void threading() throws Exception { for (int i = 0; i < 40; i++) { Future future = executorService.submit(new Callable() { @Override - public Node call() throws Exception { + public Node call() { return parser.parse(spec); } }); From 99c7c9be346bfdf28b055cdde2e963bfaa6c6813 Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 22:40:28 -0700 Subject: [PATCH 06/11] Conflitct same begin Index for different objects --- .../java/org/commonmark/test/ParserTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index a238bcdfb..269d331ea 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -347,6 +347,26 @@ public List lookup(String inline) { equalTo(" jack some")); } + @Test + public void convertLineByNodeExtensionShouldIgnoreTheNextComponentIfStartingEqualTheNextOneInTheList() { + final Node image1 = new Image(); + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + List inlinesBreakdown = new ArrayList<>(); + inlinesBreakdown.add(InlineBreakdown.of(image1, 0, 3)); + inlinesBreakdown.add(InlineBreakdown.of(new Image(), 0, 4)); + return inlinesBreakdown; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + + Node document = parser.parse("foo jack some"); + assertThat(document.getFirstChild().getFirstChild(), equalTo(image1)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext()), + equalTo(" jack some")); + } + @Test public void threading() throws Exception { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { From d33b4e5bf5fbd08fa72af883eaa10ca032b2c603 Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 22:44:04 -0700 Subject: [PATCH 07/11] Support multiples extensions --- .../java/org/commonmark/test/ParserTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index 269d331ea..b0e752f32 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -367,6 +367,62 @@ public List lookup(String inline) { equalTo(" jack some")); } + @Test + public void convertLineByNodeExtensionWithMoreThenExtension() { + final Node image1 = new Image(); + final Node image2 = new Image(); + InlineParser.NodeExtension nodeExtensionOne = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(image2, 33, 53)); + } + }; + InlineParser.NodeExtension nodeExtensionTwo = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(image1, 5, 25)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtensionOne).nodeExtension(nodeExtensionTwo).build(); + + Node document = parser.parse("some CONVERTED_TO_IMAGE_1 second CONVERTED_TO_IMAGE_2 third"); + assertThat(document.getFirstChild().getFirstChild(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild()), equalTo("some ")); + assertThat(document.getFirstChild().getFirstChild().getNext(), equalTo(image1)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext(), instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext()), + equalTo(" second ")); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext(), equalTo(image2)); + assertThat(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext(), + instanceOf(Text.class)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext().getNext().getNext().getNext()), + equalTo(" third")); + } + + @Test + public void convertLineByNodeExtensionShouldIgnoreTheNextComponentIfStartingEqualTheNextOneInTheExtensions() { + final Node image1 = new Image(); + InlineParser.NodeExtension nodeExtensionOne = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(image1, 0, 3)); + } + }; + InlineParser.NodeExtension nodeExtensionTwo = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image(), 0, 4)); + } + }; + + Parser parser = Parser.builder().nodeExtension(nodeExtensionOne).nodeExtension(nodeExtensionTwo).build(); + + Node document = parser.parse("foo jack some"); + assertThat(document.getFirstChild().getFirstChild(), equalTo(image1)); + assertThat(getLiteral(document.getFirstChild().getFirstChild().getNext()), + equalTo(" jack some")); + } + @Test public void threading() throws Exception { InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { From f3360bbdf494b5304efbbd262a032211f3edfceb Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 22:56:09 -0700 Subject: [PATCH 08/11] Usage of extension parsing random img text to Image node --- .../org/commonmark/parser/InlineParser.java | 21 +++++++++++++- .../org/commonmark/test/UsageExampleTest.java | 28 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java index 175ddc455..d9980e0a8 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java @@ -15,8 +15,17 @@ public interface InlineParser { */ void parse(String input, Node node); + /** + * Extension for inline content + */ interface NodeExtension { - List lookup(String inline); + /** + * The listing contains the all nodes that should be applied to this current line with the respective line positions + * + * @param input is the content to be parsed as custom nodes + * @return list of nodes with their location in the current line + */ + List lookup(String input); class InlineBreakdown { public static final InlineBreakdown EMPTY = @@ -43,6 +52,16 @@ public int getEndIndex() { return endIndex; } + + /** + * Inline Object to provide the {@code node} and node's line position + * @param node such as the {@code Image}, {@code Link}, etc. + * @param beginIndex as String.subString usage, the beginning index, inclusive. + * @param endIndex as String.subString usage, the ending index, exclusive. + * @exception IllegalArgumentException if the {@code endIndex} < {@code beginIndex} + * @return {@code InlineBreakdown} built, + * if {@code beginIndex} == {@code endIndex} it will considered empty object + */ public static InlineBreakdown of(Node node, int beginIndex, int endIndex) { if (endIndex < beginIndex){ throw new IllegalArgumentException("beginIndex "+ beginIndex +" must be greater then endIndex " + endIndex); diff --git a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java index 9ff646630..9d4335f8e 100644 --- a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java +++ b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java @@ -1,6 +1,7 @@ package org.commonmark.test; import org.commonmark.node.*; +import org.commonmark.parser.InlineParser; import org.commonmark.parser.Parser; import org.commonmark.renderer.NodeRenderer; import org.commonmark.renderer.html.*; @@ -12,10 +13,15 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import static java.util.Collections.singletonList; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; public class UsageExampleTest { @@ -62,6 +68,28 @@ public AttributeProvider create(AttributeProviderContext context) { renderer.render(document)); } + @Test + public void customizeNodeImageFromDifferentTextSyntax() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + @Override + public List lookup(String inline) { + return singletonList(InlineBreakdown.of(new Image("/url.png", "image"), 10, 25)); + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + HtmlRenderer renderer = HtmlRenderer.builder() + .attributeProviderFactory(new AttributeProviderFactory() { + public AttributeProvider create(AttributeProviderContext context) { + return new ImageAttributeProvider(); + } + }) + .build(); + + Node document = parser.parse("Some text ~image~/url.png"); + assertEquals("

Some text \"\"

\n", + renderer.render(document)); + } + @Test public void customizeRendering() { Parser parser = Parser.builder().build(); From c65ac9a08257d385db72b8e283ba48f19d3e2b51 Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 18 Jun 2020 23:54:21 -0700 Subject: [PATCH 09/11] Simplify construct queue object --- .../main/java/org/commonmark/internal/InlineParserImpl.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index 841bbdc55..68f5e775a 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -167,11 +167,7 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr public void parse(String content, Node block) { reset(content.trim()); - ArrayDeque customNodes = new ArrayDeque<>(); - - for (NodeExtension.InlineBreakdown inlineBreakdownNode : customNodesByExtensions()) { - customNodes.add(inlineBreakdownNode); - } + ArrayDeque customNodes = new ArrayDeque<>(customNodesByExtensions()); Node previous = null; while (true) { From 067fcf111d15617f7bf7e3f062c8a1b2a4ec5424 Mon Sep 17 00:00:00 2001 From: samukce Date: Thu, 25 Jun 2020 21:36:59 -0700 Subject: [PATCH 10/11] Add extension line usage by regex and performance test --- .../integration/ParseBenchmark.java | 67 +++++++++++++++++++ .../org/commonmark/test/UsageExampleTest.java | 38 +++++++++++ 2 files changed, 105 insertions(+) create mode 100644 commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java diff --git a/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java b/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java new file mode 100644 index 000000000..85ddb6f68 --- /dev/null +++ b/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java @@ -0,0 +1,67 @@ +package org.commonmark.integration; + +import org.commonmark.node.Image; +import org.commonmark.node.Node; +import org.commonmark.parser.InlineParser; +import org.commonmark.parser.Parser; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Measurement(batchSize = 10000, iterations = 5) +@Warmup(batchSize = 10000, iterations = 5) +public class ParseBenchmark { + private static final Parser regularParser = Parser.builder().build(); + + private static final InlineParser.NodeExtension nodeExtensionWithRegex = new InlineParser.NodeExtension() { + private final Pattern pattern = Pattern.compile("~(?[a-zA-Z]+)~(?<destination>[\\/a-zA-Z.]+)"); + + @Override + public List<InlineBreakdown> lookup(String inline) { + List<InlineBreakdown> nodesBreakDown = new ArrayList<>(); + + Matcher matcher = pattern.matcher(inline); + while (matcher.find()) { + nodesBreakDown.add(InlineBreakdown.of( + new Image(matcher.group("destination"), matcher.group("title")), + matcher.start(), + matcher.end())); + } + return nodesBreakDown; + } + }; + private static final Parser parserExtension = Parser.builder().nodeExtension(nodeExtensionWithRegex).build(); + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(".*" + ParseBenchmark.class.getSimpleName()) + .build(); + + new Runner(opt).run(); + } + + @Benchmark + public Node parseImageRegularUsage() { + return regularParser.parse("![text](/url.png)"); + } + + @Benchmark + public Node parseImageByNodeExtension() { + return parserExtension.parse("Some text ~image~/url.png"); + } +} diff --git a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java index 9d4335f8e..c0f029245 100644 --- a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java +++ b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java @@ -12,10 +12,13 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static java.util.Collections.singletonList; import static org.hamcrest.CoreMatchers.equalTo; @@ -90,6 +93,41 @@ public AttributeProvider create(AttributeProviderContext context) { renderer.render(document)); } + @Test + public void customizeNodesFromDifferentTextSyntaxUsingRegex() { + InlineParser.NodeExtension nodeExtension = new InlineParser.NodeExtension() { + Pattern pattern = Pattern.compile("~(?<title>[a-zA-Z]+)~(?<destination>[\\/a-zA-Z.]+)"); + + @Override + public List<InlineBreakdown> lookup(String inline) { + List<InlineBreakdown> nodesBreakDown = new ArrayList<>(); + + Matcher matcher = pattern.matcher(inline); + while (matcher.find()) { + nodesBreakDown.add(InlineBreakdown.of( + new Image(matcher.group("destination"), matcher.group("title")), + matcher.start(), + matcher.end())); + } + return nodesBreakDown; + } + }; + Parser parser = Parser.builder().nodeExtension(nodeExtension).build(); + HtmlRenderer renderer = HtmlRenderer.builder() + .attributeProviderFactory(new AttributeProviderFactory() { + public AttributeProvider create(AttributeProviderContext context) { + return new ImageAttributeProvider(); + } + }) + .build(); + + Node document = parser.parse("Some text ~image~/url.png anything ~SOME~/other.jpg the third one ~some~/other"); + assertEquals("<p>Some text <img src=\"/url.png\" alt=\"\" title=\"image\" class=\"border\" /> " + + "anything <img src=\"/other.jpg\" alt=\"\" title=\"SOME\" class=\"border\" /> " + + "the third one <img src=\"/other\" alt=\"\" title=\"some\" class=\"border\" /></p>\n", + renderer.render(document)); + } + @Test public void customizeRendering() { Parser parser = Parser.builder().build(); From 82748b47ec2112dc96b75da73a372a3089de5b67 Mon Sep 17 00:00:00 2001 From: samukce <samuel.p.araujo@gmail.com> Date: Thu, 25 Jun 2020 22:01:46 -0700 Subject: [PATCH 11/11] Add parser performance test with two extensions --- .../integration/ParseBenchmark.java | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java b/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java index 85ddb6f68..bac622f0f 100644 --- a/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java +++ b/commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java @@ -28,7 +28,7 @@ public class ParseBenchmark { private static final Parser regularParser = Parser.builder().build(); - private static final InlineParser.NodeExtension nodeExtensionWithRegex = new InlineParser.NodeExtension() { + private static final InlineParser.NodeExtension nodeExtensionWithRegexFirst = new InlineParser.NodeExtension() { private final Pattern pattern = Pattern.compile("~(?<title>[a-zA-Z]+)~(?<destination>[\\/a-zA-Z.]+)"); @Override @@ -45,7 +45,31 @@ public List<InlineBreakdown> lookup(String inline) { return nodesBreakDown; } }; - private static final Parser parserExtension = Parser.builder().nodeExtension(nodeExtensionWithRegex).build(); + private static final InlineParser.NodeExtension nodeExtensionWithRegexSecond = new InlineParser.NodeExtension() { + private final Pattern pattern = Pattern.compile("(?<destination>[\\/a-zA-Z.]+)\\|(?<title>[a-zA-Z]+)"); + + @Override + public List<InlineBreakdown> lookup(String inline) { + List<InlineBreakdown> nodesBreakDown = new ArrayList<>(); + + Matcher matcher = pattern.matcher(inline); + while (matcher.find()) { + nodesBreakDown.add(InlineBreakdown.of( + new Image(matcher.group("destination"), matcher.group("title")), + matcher.start(), + matcher.end())); + } + return nodesBreakDown; + } + }; + private static final Parser parserOneExtension = Parser.builder() + .nodeExtension(nodeExtensionWithRegexFirst) + .build(); + + private static final Parser parserTwoExtensions = Parser.builder() + .nodeExtension(nodeExtensionWithRegexFirst) + .nodeExtension(nodeExtensionWithRegexSecond) + .build(); public static void main(String[] args) throws RunnerException { Options opt = new OptionsBuilder() @@ -57,11 +81,16 @@ public static void main(String[] args) throws RunnerException { @Benchmark public Node parseImageRegularUsage() { - return regularParser.parse("![text](/url.png)"); + return regularParser.parse("Some text ![text](/url.png) another point ![text](/url.png)"); } @Benchmark public Node parseImageByNodeExtension() { - return parserExtension.parse("Some text ~image~/url.png"); + return parserOneExtension.parse("Some text ~image~/url.png another point ~image~/url.png"); + } + + @Benchmark + public Node parseImageByTwoNodeExtension() { + return parserTwoExtensions.parse("Some text /url.png|something another point ~image~/url.png"); } }