Add roundtrip parsing/rendering

commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java

@@ -53,4 +53,10 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
             return 0;
         }
     }
+
+
+    @Override
+    public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) {
+        return process(openingRun, closingRun);
+    }
 }

commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java

@@ -45,7 +45,7 @@ public BlockContinue tryContinue(ParserState state) {
 
     @Override
     public void addLine(SourceLine line) {
-        rowLines.add(line);
+        rowLines.add(line.getLiteralLine());
     }
 
     @Override

commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java

@@ -85,4 +85,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
 
         return 1;
     }
+
+    @Override
+    public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) {
+        return process(openingRun, closingRun);
+    }
 }

commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java

@@ -53,4 +53,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
             return 0;
         }
     }
+
+    @Override
+    public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) {
+        return process(openingRun, closingRun);
+    }
 }

commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java

@@ -9,6 +9,16 @@ public class BlockQuoteParser extends AbstractBlockParser {
 
     private final BlockQuote block = new BlockQuote();
 
+    // Preserve original default constructor by explicitly defining one
+    public BlockQuoteParser() {
+        super();
+    }
+
+    public BlockQuoteParser(String preMarkerWhitespace, String postMarkerWhitespace) {
+        block.setPreMarkerWhitespace(preMarkerWhitespace);
+        block.setPostMarkerWhitespace(postMarkerWhitespace);
+    }
+
     @Override
     public boolean isContainer() {
         return true;
@@ -47,13 +57,24 @@ private static boolean isMarker(ParserState state, int index) {
     public static class Factory extends AbstractBlockParserFactory {
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
             int nextNonSpace = state.getNextNonSpaceIndex();
+            String preMarkerWhitespace = "";
+
+            if(nextNonSpace > 0) {
+                preMarkerWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace - 1, 0);
+            }
+
             if (isMarker(state, nextNonSpace)) {
                 int newColumn = state.getColumn() + state.getIndent() + 1;
+
                 // optional following space or tab
+                String postMarkerWhitespace = "";
+
                 if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) {
+                    postMarkerWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), nextNonSpace + 1, state.getLine().getContent().length() - 1);
                     newColumn++;
                 }
-                return BlockStart.of(new BlockQuoteParser()).atColumn(newColumn);
+
+                return BlockStart.of(new BlockQuoteParser(preMarkerWhitespace, postMarkerWhitespace)).atColumn(newColumn);
             } else {
                 return BlockStart.none();
             }

commonmark/src/main/java/org/commonmark/internal/DocumentParser.java

@@ -1,15 +1,43 @@
 package org.commonmark.internal;
 
-import org.commonmark.internal.util.Parsing;
-import org.commonmark.node.*;
-import org.commonmark.parser.*;
-import org.commonmark.parser.block.*;
-import org.commonmark.parser.delimiter.DelimiterProcessor;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.Reader;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.commonmark.internal.inline.BlankLineParser;
+import org.commonmark.internal.util.Parsing;
+import org.commonmark.node.Block;
+import org.commonmark.node.BlockQuote;
+import org.commonmark.node.Document;
+import org.commonmark.node.FencedCodeBlock;
+import org.commonmark.node.Heading;
+import org.commonmark.node.HtmlBlock;
+import org.commonmark.node.IndentedCodeBlock;
+import org.commonmark.node.LinkReferenceDefinition;
+import org.commonmark.node.ListBlock;
+import org.commonmark.node.Paragraph;
+import org.commonmark.node.SourceSpan;
+import org.commonmark.node.ThematicBreak;
+import org.commonmark.parser.IncludeSourceSpans;
+import org.commonmark.parser.InlineParser;
+import org.commonmark.parser.InlineParserFactory;
+import org.commonmark.parser.SourceLine;
+import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.block.BlockContinue;
+import org.commonmark.parser.block.BlockParser;
+import org.commonmark.parser.block.BlockParserFactory;
+import org.commonmark.parser.block.BlockStart;
+import org.commonmark.parser.block.MatchedBlockParser;
+import org.commonmark.parser.block.ParserState;
+import org.commonmark.parser.delimiter.DelimiterProcessor;
 
 public class DocumentParser implements ParserState {
 
@@ -275,13 +303,19 @@ private void parseLine(CharSequence ln) {
             addLine();
 
         } else {
-
             // finalize any blocks not matched
             if (unmatchedBlocks > 0) {
                 closeBlockParsers(unmatchedBlocks);
             }
 
             if (!blockParser.isContainer()) {
+                // Capture any blank lines within a separate node for roundtrip purposes
+                // BlankLine nodes do _not_ interrupt the current node type's processor
+                if(isBlank()) {
+                    BlankLineParser blankLineParser = new BlankLineParser(ln.toString());
+                    addChild(new OpenBlockParser(blankLineParser, lastIndex));
+                }
+
                 addLine();
             } else if (!isBlank()) {
                 // create paragraph container for line
@@ -297,6 +331,13 @@ private void parseLine(CharSequence ln) {
                 //
                 // The first line does not start a paragraph yet, but we still want to record source positions.
                 addSourceSpans();
+
+                // Capture any blank lines within a separate node for roundtrip purposes
+                // BlankLine nodes do _not_ interrupt the current node type's processor
+                if(isBlank()) {
+                    BlankLineParser blankLineParser = new BlankLineParser(ln.toString());
+                    addChild(new OpenBlockParser(blankLineParser, lastIndex));
+                }
             }
         }
     }
@@ -403,18 +444,23 @@ private void addLine() {
             }
             sb.append(rest);
             content = sb.toString();
-        } else if (index == 0) {
-            content = line.getContent();
         } else {
-            content = line.getContent().subSequence(index, line.getContent().length());
+            content = line.getContent().toString();
         }
+
         SourceSpan sourceSpan = null;
         if (includeSourceSpans == IncludeSourceSpans.BLOCKS_AND_INLINES) {
             // Note that if we're in a partially-consumed tab, the length here corresponds to the content but not to the
             // actual source length. That sounds like a problem, but I haven't found a test case where it matters (yet).
-            sourceSpan = SourceSpan.of(lineIndex, index, content.length());
+            sourceSpan = SourceSpan.of(lineIndex, index, line.getLiteralLine().getContent().length());
         }
-        getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan));
+
+        if(!columnIsInTab) {
+            getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, index));
+        }else {
+            getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, 0));
+        }
+
         addSourceSpans();
     }
 
@@ -425,6 +471,11 @@ private void addSourceSpans() {
                 OpenBlockParser openBlockParser = openBlockParsers.get(i);
                 int blockIndex = openBlockParser.sourceIndex;
                 int length = line.getContent().length() - blockIndex;
+
+                if(line.getLiteralIndex() != 0) {
+                    length = line.getLiteralLine().getContent().length() - blockIndex;
+                }
+
                 if (length != 0) {
                     openBlockParser.blockParser.addSourceSpan(SourceSpan.of(lineIndex, blockIndex, length));
                 }
@@ -481,12 +532,21 @@ private void processInlines() {
      * its parent, and so on until we find a block that can accept children.
      */
     private void addChild(OpenBlockParser openBlockParser) {
-        while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) {
-            closeBlockParsers(1);
+        // BlankLine nodes do _not_ interrupt the current node type's processor, as
+        //    they are non-standard CommonMark nodes meant only for roundtrip processing
+        if(!(openBlockParser.blockParser instanceof BlankLineParser)) {
+            while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) {
+                closeBlockParsers(1);
+            }
         }
 
         getActiveBlockParser().getBlock().appendChild(openBlockParser.blockParser.getBlock());
-        activateBlockParser(openBlockParser);
+
+        // BlankLine nodes do _not_ become the active parser, as this would interrupt the
+        //    parser before it
+        if(!(openBlockParser.blockParser instanceof BlankLineParser)) {
+            activateBlockParser(openBlockParser);
+        }
     }
 
     private void activateBlockParser(OpenBlockParser openBlockParser) {
@@ -520,6 +580,16 @@ private Block prepareActiveBlockParserForReplacement() {
     private Document finalizeAndProcess() {
         closeBlockParsers(openBlockParsers.size());
         processInlines();
+        Document docNode = documentBlockParser.getBlock();
+        String whitespaceEndOfDocument = "";
+
+        // CommonMark test cases end with newlines, indicating that this should be standard
+        //    in roundtrip scenarios. If a post-document whitespace doesn't exist, add it.
+        if(whitespaceEndOfDocument.isEmpty()) {
+            whitespaceEndOfDocument = "\n";
+        }
+
+        docNode.setEndOfDocumentWhitespace(whitespaceEndOfDocument);
         return documentBlockParser.getBlock();
     }
 

commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java

@@ -4,21 +4,28 @@
 import org.commonmark.node.Block;
 import org.commonmark.node.FencedCodeBlock;
 import org.commonmark.parser.SourceLine;
-import org.commonmark.parser.block.*;
-
-import static org.commonmark.internal.util.Escaping.unescapeString;
+import org.commonmark.parser.block.AbstractBlockParser;
+import org.commonmark.parser.block.AbstractBlockParserFactory;
+import org.commonmark.parser.block.BlockContinue;
+import org.commonmark.parser.block.BlockStart;
+import org.commonmark.parser.block.MatchedBlockParser;
+import org.commonmark.parser.block.ParserState;
 
 public class FencedCodeBlockParser extends AbstractBlockParser {
 
     private final FencedCodeBlock block = new FencedCodeBlock();
 
     private String firstLine;
     private StringBuilder otherLines = new StringBuilder();
+    private StringBuilder rawLines = new StringBuilder();
 
     public FencedCodeBlockParser(char fenceChar, int fenceLength, int fenceIndent) {
         block.setFenceChar(fenceChar);
-        block.setFenceLength(fenceLength);
-        block.setFenceIndent(fenceIndent);
+        block.setStartFenceLength(fenceLength);
+
+        // Fenced code blocks can't be indented by newlines or tabs, so it's safe to assume
+        //    spaces for the indentation
+        block.setPreStartFenceWhitespace(Parsing.generateSpaces(fenceIndent));
     }
 
     @Override
@@ -32,11 +39,20 @@ public BlockContinue tryContinue(ParserState state) {
         int newIndex = state.getIndex();
         CharSequence line = state.getLine().getContent();
         if (state.getIndent() < Parsing.CODE_BLOCK_INDENT && nextNonSpace < line.length() && line.charAt(nextNonSpace) == block.getFenceChar() && isClosing(line, nextNonSpace)) {
+            // Capture whitespace before closing fence (if any) for roundtrip purposes
+            if(nextNonSpace > 0) {
+                block.setPreEndFenceWhitespace(Parsing.collectWhitespace(line, 0, nextNonSpace));
+            }
+
             // closing fence - we're at end of line, so we can finalize now
             return BlockContinue.finished();
         } else {
+            // Capture line before optional spaces are removed
+            rawLines.append(line.toString());
+            rawLines.append('\n');
+
             // skip optional spaces of fence indent
-            int i = block.getFenceIndent();
+            int i = block.getStartFenceIndent();
             int length = line.length();
             while (i > 0 && newIndex < length && line.charAt(newIndex) == ' ') {
                 newIndex++;
@@ -49,17 +65,47 @@ public BlockContinue tryContinue(ParserState state) {
     @Override
     public void addLine(SourceLine line) {
         if (firstLine == null) {
-            firstLine = line.getContent().toString();
+
+            // Raw line is the same as literal line, so just use it
+            if(line.getLiteralIndex() == 0) {
+                firstLine = line.getContent().toString();
+            }
+            // Raw line differs from literal line, so preserve literal line
+            else {
+                firstLine = line.substring(line.getLiteralIndex(), line.getContent().length()).getContent().toString();
+            }
         } else {
-            otherLines.append(line.getContent());
+            // Raw line is the same as literal line, so just use it
+            if(line.getLiteralIndex() == 0) {
+                otherLines.append(line.getContent());
+            }
+            // Raw line differs from literal line, so preserve literal line
+            else {
+                otherLines.append(line.getLiteralLine().getContent());
+            }
             otherLines.append('\n');
         }
     }
 
     @Override
     public void closeBlock() {
         // first line becomes info string
-        block.setInfo(unescapeString(firstLine.trim()));
+        // It is not trimmed or escaped here to allow roundtrip processing
+        block.setInfo(firstLine);
+
+        // Remove final newline in raw content, if it exists
+        if(rawLines != null && rawLines.length() != 0) {
+            if(rawLines.charAt(rawLines.length() - 1) == '\n') {
+                rawLines.deleteCharAt(rawLines.length() - 1);
+            }
+        }
+
+        // Capture raw content (no indentation removed) between end of info string
+        //    and beginning of end fence
+        block.setRaw(rawLines.toString());
+
+        // Capture content (without indentation) between end of info string
+        //    and beginning of end fence
         block.setLiteral(otherLines.toString());
     }
 
@@ -72,10 +118,14 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
                 return BlockStart.none();
             }
 
+            if(indent == 0) {
+                indent = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()).length();
+            }
+
             int nextNonSpace = state.getNextNonSpaceIndex();
             FencedCodeBlockParser blockParser = checkOpener(state.getLine().getContent(), nextNonSpace, indent);
             if (blockParser != null) {
-                return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getFenceLength());
+                return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getStartFenceLength());
             } else {
                 return BlockStart.none();
             }
@@ -120,13 +170,22 @@ private static FencedCodeBlockParser checkOpener(CharSequence line, int index, i
     // code fence.
     private boolean isClosing(CharSequence line, int index) {
         char fenceChar = block.getFenceChar();
-        int fenceLength = block.getFenceLength();
+        int fenceLength = block.getStartFenceLength();
         int fences = Parsing.skip(fenceChar, line, index, line.length()) - index;
         if (fences < fenceLength) {
             return false;
+        }else {
+            block.setEndFenceLength(fences);
         }
+
         // spec: The closing code fence [...] may be followed only by spaces, which are ignored.
         int after = Parsing.skipSpaceTab(line, index + fences, line.length());
+
+        // Capture post-fence spaces (if any) for roundtrip purposes
+        if(after > 0 && after != (index + fences)) {
+            block.setPostBlockWhitespace(Parsing.collectWhitespaceBackwards(line, line.length() - 1, 0));
+        }
+
         return after == line.length();
     }
 }