Inline parsing extensible

commonmark-integration-test/src/test/java/org/commonmark/integration/ParseBenchmark.java

@@ -0,0 +1,96 @@
+package org.commonmark.integration;
+
+import org.commonmark.node.Image;
+import org.commonmark.node.Node;
+import org.commonmark.parser.InlineParser;
+import org.commonmark.parser.Parser;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@BenchmarkMode(Mode.SingleShotTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Measurement(batchSize = 10000, iterations = 5)
+@Warmup(batchSize = 10000, iterations = 5)
+public class ParseBenchmark {
+    private static final Parser regularParser = Parser.builder().build();
+
+    private static final InlineParser.NodeExtension nodeExtensionWithRegexFirst = new InlineParser.NodeExtension() {
+        private final Pattern pattern = Pattern.compile("~(?<title>[a-zA-Z]+)~(?<destination>[\\/a-zA-Z.]+)");
+
+        @Override
+        public List<InlineBreakdown> lookup(String inline) {
+            List<InlineBreakdown> nodesBreakDown = new ArrayList<>();
+
+            Matcher matcher = pattern.matcher(inline);
+            while (matcher.find()) {
+                nodesBreakDown.add(InlineBreakdown.of(
+                        new Image(matcher.group("destination"), matcher.group("title")),
+                        matcher.start(),
+                        matcher.end()));
+            }
+            return nodesBreakDown;
+        }
+    };
+    private static final InlineParser.NodeExtension nodeExtensionWithRegexSecond = new InlineParser.NodeExtension() {
+        private final Pattern pattern = Pattern.compile("(?<destination>[\\/a-zA-Z.]+)\\|(?<title>[a-zA-Z]+)");
+
+        @Override
+        public List<InlineBreakdown> lookup(String inline) {
+            List<InlineBreakdown> nodesBreakDown = new ArrayList<>();
+
+            Matcher matcher = pattern.matcher(inline);
+            while (matcher.find()) {
+                nodesBreakDown.add(InlineBreakdown.of(
+                        new Image(matcher.group("destination"), matcher.group("title")),
+                        matcher.start(),
+                        matcher.end()));
+            }
+            return nodesBreakDown;
+        }
+    };
+    private static final Parser parserOneExtension = Parser.builder()
+            .nodeExtension(nodeExtensionWithRegexFirst)
+            .build();
+
+    private static final Parser parserTwoExtensions = Parser.builder()
+            .nodeExtension(nodeExtensionWithRegexFirst)
+            .nodeExtension(nodeExtensionWithRegexSecond)
+            .build();
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(".*" + ParseBenchmark.class.getSimpleName())
+                .build();
+
+        new Runner(opt).run();
+    }
+
+    @Benchmark
+    public Node parseImageRegularUsage() {
+        return regularParser.parse("Some text ![text](/url.png) another point ![text](/url.png)");
+    }
+
+    @Benchmark
+    public Node parseImageByNodeExtension() {
+        return parserOneExtension.parse("Some text ~image~/url.png another point ~image~/url.png");
+    }
+
+    @Benchmark
+    public Node parseImageByTwoNodeExtension() {
+        return parserTwoExtensions.parse("Some text /url.png|something another point ~image~/url.png");
+    }
+}

commonmark/src/main/java/org/commonmark/internal/DocumentParser.java

@@ -63,6 +63,7 @@ public class DocumentParser implements ParserState {
     private final List<BlockParserFactory> blockParserFactories;
     private final InlineParserFactory inlineParserFactory;
     private final List<DelimiterProcessor> delimiterProcessors;
+    private final List<InlineParser.NodeExtension> nodeExtensions;
     private final DocumentBlockParser documentBlockParser;
     private final Map<String, LinkReferenceDefinition> definitions = new LinkedHashMap<>();
 
@@ -71,10 +72,12 @@ public class DocumentParser implements ParserState {
     private Set<BlockParser> allBlockParsers = new LinkedHashSet<>();
 
     public DocumentParser(List<BlockParserFactory> blockParserFactories, InlineParserFactory inlineParserFactory,
-                          List<DelimiterProcessor> delimiterProcessors) {
+                          List<DelimiterProcessor> delimiterProcessors,
+                          List<InlineParser.NodeExtension> nodeExtensions) {
         this.blockParserFactories = blockParserFactories;
         this.inlineParserFactory = inlineParserFactory;
         this.delimiterProcessors = delimiterProcessors;
+        this.nodeExtensions = nodeExtensions;
 
         this.documentBlockParser = new DocumentBlockParser();
         activateBlockParser(this.documentBlockParser);
@@ -415,7 +418,7 @@ private void addDefinitionsFrom(ParagraphParser paragraphParser) {
      * Walk through a block & children recursively, parsing string content into inline content where appropriate.
      */
     private void processInlines() {
-        InlineParserContextImpl context = new InlineParserContextImpl(delimiterProcessors, definitions);
+        InlineParserContextImpl context = new InlineParserContextImpl(delimiterProcessors, definitions, nodeExtensions);
         InlineParser inlineParser = inlineParserFactory.create(context);
 
         for (BlockParser blockParser : allBlockParsers) {

commonmark/src/main/java/org/commonmark/internal/InlineParserContextImpl.java

@@ -1,6 +1,7 @@
 package org.commonmark.internal;
 
 import org.commonmark.node.LinkReferenceDefinition;
+import org.commonmark.parser.InlineParser;
 import org.commonmark.parser.InlineParserContext;
 import org.commonmark.parser.delimiter.DelimiterProcessor;
 
@@ -11,11 +12,14 @@ public class InlineParserContextImpl implements InlineParserContext {
 
     private final List<DelimiterProcessor> delimiterProcessors;
     private final Map<String, LinkReferenceDefinition> linkReferenceDefinitions;
+    private final List<InlineParser.NodeExtension> nodeExtensions;
 
     public InlineParserContextImpl(List<DelimiterProcessor> delimiterProcessors,
-                                   Map<String, LinkReferenceDefinition> linkReferenceDefinitions) {
+                                   Map<String, LinkReferenceDefinition> linkReferenceDefinitions,
+                                   List<InlineParser.NodeExtension> nodeExtensions) {
         this.delimiterProcessors = delimiterProcessors;
         this.linkReferenceDefinitions = linkReferenceDefinitions;
+        this.nodeExtensions = nodeExtensions;
     }
 
     @Override
@@ -27,4 +31,9 @@ public List<DelimiterProcessor> getCustomDelimiterProcessors() {
     public LinkReferenceDefinition getLinkReferenceDefinition(String label) {
         return linkReferenceDefinitions.get(label);
     }
+
+    @Override
+    public List<InlineParser.NodeExtension> nodeExtensions() {
+        return nodeExtensions;
+    }
 }

commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java

@@ -6,12 +6,29 @@
 import org.commonmark.internal.util.Html5Entities;
 import org.commonmark.internal.util.LinkScanner;
 import org.commonmark.internal.util.Parsing;
-import org.commonmark.node.*;
+import org.commonmark.node.Code;
+import org.commonmark.node.HardLineBreak;
+import org.commonmark.node.HtmlInline;
+import org.commonmark.node.Image;
+import org.commonmark.node.Link;
+import org.commonmark.node.LinkReferenceDefinition;
+import org.commonmark.node.Node;
+import org.commonmark.node.SoftLineBreak;
+import org.commonmark.node.Text;
 import org.commonmark.parser.InlineParser;
 import org.commonmark.parser.InlineParserContext;
 import org.commonmark.parser.delimiter.DelimiterProcessor;
 
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -56,6 +73,7 @@ public class InlineParserImpl implements InlineParser {
     private final BitSet delimiterCharacters;
     private final Map<Character, DelimiterProcessor> delimiterProcessors;
     private final InlineParserContext context;
+    private final List<NodeExtension> nodeExtensions;
 
     private String input;
     private int index;
@@ -77,6 +95,7 @@ public InlineParserImpl(InlineParserContext inlineParserContext) {
         this.specialCharacters = calculateSpecialCharacters(delimiterCharacters);
 
         this.context = inlineParserContext;
+        this.nodeExtensions = inlineParserContext.nodeExtensions();
     }
 
     public static BitSet calculateDelimiterCharacters(Set<Character> characters) {
@@ -148,9 +167,11 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr
     public void parse(String content, Node block) {
         reset(content.trim());
 
+        ArrayDeque<NodeExtension.InlineBreakdown> customNodes = new ArrayDeque<>(customNodesByExtensions());
+
         Node previous = null;
         while (true) {
-            Node node = parseInline(previous);
+            Node node = parseInline(previous, customNodes);
             previous = node;
             if (node != null) {
                 block.appendChild(node);
@@ -184,7 +205,22 @@ private Text text(String text) {
      * On success, return the new inline node.
      * On failure, return null.
      */
-    private Node parseInline(Node previous) {
+    private Node parseInline(Node previous, ArrayDeque<NodeExtension.InlineBreakdown> customNodes) {
+        NodeExtension.InlineBreakdown inlineBreakdown = customNodes.isEmpty()
+                ? NodeExtension.InlineBreakdown.EMPTY
+                : customNodes.getFirst();
+        int nextBeginIndexForCustomNode = inlineBreakdown.getBeginIndex();
+
+        if (inlineBreakdown.getBeginIndex() == index) {
+            while (index < inlineBreakdown.getEndIndex()) {
+                index++;
+            }
+            return customNodes.pollFirst().getNode();
+        } else if (inlineBreakdown.getBeginIndex() < index) {
+            customNodes.pollFirst();
+            nextBeginIndexForCustomNode = input.length();
+        }
+
         char c = peek();
         if (c == '\0') {
             return null;
@@ -225,7 +261,7 @@ private Node parseInline(Node previous) {
                     DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c);
                     node = parseDelimiters(delimiterProcessor, c);
                 } else {
-                    node = parseString();
+                    node = parseString(Math.min(nextBeginIndexForCustomNode, input.length()));
                 }
                 break;
         }
@@ -384,6 +420,25 @@ private Node parseDelimiters(DelimiterProcessor delimiterProcessor, char delimit
         return node;
     }
 
+    private List<NodeExtension.InlineBreakdown> customNodesByExtensions() {
+        List<NodeExtension.InlineBreakdown> nodes = new ArrayList<>();
+
+        for (NodeExtension nodeExtension : nodeExtensions) {
+            List<NodeExtension.InlineBreakdown> inlineBreakdowns = nodeExtension.lookup(input);
+            if (inlineBreakdowns != null) {
+                nodes.addAll(inlineBreakdowns);
+            }
+        }
+
+        Collections.sort(nodes, new Comparator<NodeExtension.InlineBreakdown>() {
+            @Override
+            public int compare(NodeExtension.InlineBreakdown node1, NodeExtension.InlineBreakdown node2) {
+                return Integer.compare(node1.getBeginIndex(), node2.getBeginIndex());
+            }
+        });
+        return nodes;
+    }
+
     /**
      * Add open bracket to delimiter stack and add a text node to block's children.
      */
@@ -652,10 +707,10 @@ private Node parseEntity() {
 
     /**
      * Parse a run of ordinary characters, or a single character with a special meaning in markdown, as a plain string.
+     * @param length
      */
-    private Node parseString() {
+    private Node parseString(int length) {
         int begin = index;
-        int length = input.length();
         while (index != length) {
             if (specialCharacters.get(input.charAt(index))) {
                 break;

commonmark/src/main/java/org/commonmark/parser/InlineParser.java

@@ -2,6 +2,8 @@
 
 import org.commonmark.node.Node;
 
+import java.util.List;
+
 /**
  * Parser for inline content (text, links, emphasized text, etc).
  */
@@ -12,4 +14,65 @@ public interface InlineParser {
      * @param node the node to append resulting nodes to (as children)
      */
     void parse(String input, Node node);
+
+    /**
+     *  Extension for inline content
+     */
+    interface NodeExtension {
+        /**
+         * The listing contains the all nodes that should be applied to this current line with the respective line positions
+         *
+         * @param input is the content to be parsed as custom nodes
+         * @return list of nodes with their location in the current line
+         */
+        List<InlineBreakdown> lookup(String input);
+
+        class InlineBreakdown {
+            public static final InlineBreakdown EMPTY =
+                    new InlineBreakdown(null, Integer.MAX_VALUE, Integer.MAX_VALUE);
+            private final Node node;
+            private final int beginIndex;
+            private final int endIndex;
+
+            private InlineBreakdown(Node node, int beginIndex, int endIndex) {
+                this.node = node;
+                this.beginIndex = beginIndex;
+                this.endIndex = endIndex;
+            }
+
+            public Node getNode() {
+                return this.node;
+            }
+
+            public int getBeginIndex() {
+                return beginIndex;
+            }
+
+            public int getEndIndex() {
+                return endIndex;
+            }
+
+
+            /**
+             * Inline Object to provide the {@code node} and node's line position
+             * @param node such as the {@code Image}, {@code Link}, etc.
+             * @param beginIndex as String.subString usage, the beginning index, inclusive.
+             * @param endIndex as String.subString usage, the ending index, exclusive.
+             * @exception  IllegalArgumentException  if the {@code endIndex} < {@code beginIndex}
+             * @return {@code InlineBreakdown} built,
+             *          if {@code beginIndex} == {@code endIndex} it will considered empty object
+             */
+            public static InlineBreakdown of(Node node, int beginIndex, int endIndex) {
+                if (endIndex < beginIndex){
+                    throw new IllegalArgumentException("beginIndex "+ beginIndex +" must be greater then endIndex " + endIndex);
+                }
+
+                if (beginIndex == endIndex) {
+                    return EMPTY;
+                }
+
+                return new InlineBreakdown(node, beginIndex, endIndex);
+            }
+        }
+    }
 }

commonmark/src/main/java/org/commonmark/parser/InlineParserContext.java

@@ -22,4 +22,6 @@ public interface InlineParserContext {
      * @return the definition if one exists, {@code null} otherwise
      */
     LinkReferenceDefinition getLinkReferenceDefinition(String label);
+
+    List<InlineParser.NodeExtension> nodeExtensions();
 }