Inline parsing overhaul WIP

commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java

@@ -1,20 +1,11 @@
 package org.commonmark.ext.gfm.tables.internal;
 
-import org.commonmark.ext.gfm.tables.TableBlock;
-import org.commonmark.ext.gfm.tables.TableBody;
-import org.commonmark.ext.gfm.tables.TableCell;
-import org.commonmark.ext.gfm.tables.TableHead;
-import org.commonmark.ext.gfm.tables.TableRow;
+import org.commonmark.ext.gfm.tables.*;
 import org.commonmark.node.Block;
 import org.commonmark.node.Node;
 import org.commonmark.node.SourceSpan;
 import org.commonmark.parser.InlineParser;
-import org.commonmark.parser.block.AbstractBlockParser;
-import org.commonmark.parser.block.AbstractBlockParserFactory;
-import org.commonmark.parser.block.BlockContinue;
-import org.commonmark.parser.block.BlockStart;
-import org.commonmark.parser.block.MatchedBlockParser;
-import org.commonmark.parser.block.ParserState;
+import org.commonmark.parser.block.*;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -116,7 +107,7 @@ private TableCell parseCell(CellSource cell, int column, InlineParser inlinePars
             tableCell.setSourceSpans(Collections.singletonList(cell.sourceSpan));
         }
 
-        inlineParser.parse(cell.content, tableCell);
+        inlineParser.parse(Collections.<CharSequence>singletonList(cell.content.trim()), tableCell);
 
         return tableCell;
     }
@@ -246,11 +237,12 @@ public static class Factory extends AbstractBlockParserFactory {
         @Override
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
             CharSequence line = state.getLine();
-            CharSequence paragraph = matchedBlockParser.getParagraphContent();
-            if (paragraph != null && paragraph.toString().contains("|") && !paragraph.toString().contains("\n")) {
+            List<CharSequence> paragraphLines = matchedBlockParser.getParagraphLines();
+            if (paragraphLines.size() == 1 && paragraphLines.get(0).toString().contains("|")) {
                 CharSequence separatorLine = line.subSequence(state.getIndex(), line.length());
                 List<TableCell.Alignment> columns = parseSeparator(separatorLine);
                 if (columns != null && !columns.isEmpty()) {
+                    CharSequence paragraph = paragraphLines.get(0);
                     List<CellSource> headerCells = split(paragraph, null);
                     if (columns.size() >= headerCells.size()) {
                         return BlockStart.of(new TableBlockParser(columns, paragraph))

commonmark/src/main/java/org/commonmark/internal/Bracket.java

@@ -1,5 +1,6 @@
 package org.commonmark.internal;
 
+import org.commonmark.internal.inline.Position;
 import org.commonmark.node.Text;
 
 /**
@@ -8,7 +9,10 @@
 public class Bracket {
 
     public final Text node;
-    public final int index;
+    /**
+     * The position of the content (after the opening bracket)
+     */
+    public final Position contentPosition;
     public final boolean image;
 
     /**
@@ -31,17 +35,17 @@ public class Bracket {
      */
     public boolean bracketAfter = false;
 
-    static public Bracket link(Text node, int index, Bracket previous, Delimiter previousDelimiter) {
-        return new Bracket(node, index, previous, previousDelimiter, false);
+    static public Bracket link(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter) {
+        return new Bracket(node, contentPosition, previous, previousDelimiter, false);
     }
 
-    static public Bracket image(Text node, int index, Bracket previous, Delimiter previousDelimiter) {
-        return new Bracket(node, index, previous, previousDelimiter, true);
+    static public Bracket image(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter) {
+        return new Bracket(node, contentPosition, previous, previousDelimiter, true);
     }
 
-    private Bracket(Text node, int index, Bracket previous, Delimiter previousDelimiter, boolean image) {
+    private Bracket(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter, boolean image) {
         this.node = node;
-        this.index = index;
+        this.contentPosition = contentPosition;
         this.image = image;
         this.previous = previous;
         this.previousDelimiter = previousDelimiter;

commonmark/src/main/java/org/commonmark/internal/DocumentParser.java

@@ -425,6 +425,8 @@ private void addLine() {
             }
             sb.append(rest);
             content = sb.toString();
+        } else if (index == 0) {
+            content = line;
         } else {
             content = line.subSequence(index, line.length());
         }
@@ -458,9 +460,8 @@ private BlockStartImpl findBlockStart(BlockParser blockParser) {
     }
 
     /**
-     * Finalize a block. Close it and do any necessary postprocessing, e.g. creating string_content from strings,
-     * setting the 'tight' or 'loose' status of a list, and parsing the beginnings of paragraphs for reference
-     * definitions.
+     * Finalize a block. Close it and do any necessary postprocessing, e.g. setting the content of blocks and
+     * collecting link reference definitions from paragraphs.
      */
     private void finalize(BlockParser blockParser) {
         if (blockParser instanceof ParagraphParser) {
@@ -567,17 +568,12 @@ public BlockParser getMatchedBlockParser() {
         }
 
         @Override
-        public CharSequence getParagraphContent() {
+        public List<CharSequence> getParagraphLines() {
             if (matchedBlockParser instanceof ParagraphParser) {
                 ParagraphParser paragraphParser = (ParagraphParser) matchedBlockParser;
-                CharSequence content = paragraphParser.getContentString();
-                if (content.length() == 0) {
-                    return null;
-                }
-
-                return content;
+                return Collections.unmodifiableList(paragraphParser.getParagraphLines());
             }
-            return null;
+            return Collections.emptyList();
         }
     }
 

commonmark/src/main/java/org/commonmark/internal/HeadingParser.java

@@ -6,12 +6,15 @@
 import org.commonmark.parser.InlineParser;
 import org.commonmark.parser.block.*;
 
+import java.util.Collections;
+import java.util.List;
+
 public class HeadingParser extends AbstractBlockParser {
 
     private final Heading block = new Heading();
-    private final String content;
+    private final List<CharSequence> content;
 
-    public HeadingParser(int level, String content) {
+    public HeadingParser(int level, List<CharSequence> content) {
         block.setLevel(level);
         this.content = content;
     }
@@ -49,10 +52,9 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
 
             int setextHeadingLevel = getSetextHeadingLevel(line, nextNonSpace);
             if (setextHeadingLevel > 0) {
-                CharSequence paragraph = matchedBlockParser.getParagraphContent();
-                if (paragraph != null) {
-                    String content = paragraph.toString();
-                    return BlockStart.of(new HeadingParser(setextHeadingLevel, content))
+                List<CharSequence> paragraph = matchedBlockParser.getParagraphLines();
+                if (!paragraph.isEmpty()) {
+                    return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph))
                             .atIndex(line.length())
                             .replaceActiveBlockParser();
                 }
@@ -73,25 +75,29 @@ private static HeadingParser getAtxHeading(CharSequence line, int index) {
             return null;
         }
 
-        int start = index + level;
-        if (start >= line.length()) {
+        int afterMarker = index + level;
+        if (afterMarker >= line.length()) {
             // End of line after markers is an empty heading
-            return new HeadingParser(level, "");
+            return new HeadingParser(level, Collections.<CharSequence>emptyList());
         }
 
-        char next = line.charAt(start);
+        char next = line.charAt(afterMarker);
         if (!(next == ' ' || next == '\t')) {
             return null;
         }
 
-        int beforeSpace = Parsing.skipSpaceTabBackwards(line, line.length() - 1, start);
-        int beforeHash = Parsing.skipBackwards('#', line, beforeSpace, start);
-        int beforeTrailer = Parsing.skipSpaceTabBackwards(line, beforeHash, start);
-        if (beforeTrailer != beforeHash) {
-            return new HeadingParser(level, line.subSequence(start, beforeTrailer + 1).toString());
-        } else {
-            return new HeadingParser(level, line.subSequence(start, beforeSpace + 1).toString());
+        int start = Parsing.skipSpaceTab(line, afterMarker, line.length());
+
+        int beforeSpace = Parsing.skipSpaceTabBackwards(line, line.length() - 1, afterMarker);
+        int beforeHash = Parsing.skipBackwards('#', line, beforeSpace, afterMarker);
+        int beforeTrailer = Parsing.skipSpaceTabBackwards(line, beforeHash, afterMarker);
+        // Trailing `#` need to be separated with at least one space/tab, otherwise they are part of the content.
+        int end = (beforeTrailer < beforeHash) ? beforeTrailer + 1 : beforeSpace + 1;
+        if (start >= end) {
+            // Empty, e.g. `### ###`
+            return new HeadingParser(level, Collections.<CharSequence>emptyList());
         }
+        return new HeadingParser(level, Collections.singletonList(line.subSequence(start, end)));
     }
 
     // spec: A setext heading underline is a sequence of = characters or a sequence of - characters, with no more than