diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java index 7d54eedf2..de2073487 100644 --- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java +++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java @@ -53,4 +53,10 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 0; } } + + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java index a8eedb7a6..b978ef95b 100644 --- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java +++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java @@ -45,7 +45,7 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - rowLines.add(line); + rowLines.add(line.getLiteralLine()); } @Override diff --git a/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java b/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java index a584948e3..9f3b6e9bb 100644 --- a/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java +++ b/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java @@ -85,4 +85,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java index b0bfb4c6e..b112f510f 100644 --- a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java +++ b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java @@ -53,4 +53,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 0; } } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java index 00cdbc11e..901d6a04e 100644 --- a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java @@ -9,6 +9,16 @@ public class BlockQuoteParser extends AbstractBlockParser { private final BlockQuote block = new BlockQuote(); + // Preserve original default constructor by explicitly defining one + public BlockQuoteParser() { + super(); + } + + public BlockQuoteParser(String preMarkerWhitespace, String postMarkerWhitespace) { + block.setPreMarkerWhitespace(preMarkerWhitespace); + block.setPostMarkerWhitespace(postMarkerWhitespace); + } + @Override public boolean isContainer() { return true; @@ -47,13 +57,24 @@ private static boolean isMarker(ParserState state, int index) { public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { int nextNonSpace = state.getNextNonSpaceIndex(); + String preMarkerWhitespace = ""; + + if(nextNonSpace > 0) { + preMarkerWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace - 1, 0); + } + if (isMarker(state, nextNonSpace)) { int newColumn = state.getColumn() + state.getIndent() + 1; + // optional following space or tab + String postMarkerWhitespace = ""; + if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) { + postMarkerWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), nextNonSpace + 1, state.getLine().getContent().length() - 1); newColumn++; } - return BlockStart.of(new BlockQuoteParser()).atColumn(newColumn); + + return BlockStart.of(new BlockQuoteParser(preMarkerWhitespace, postMarkerWhitespace)).atColumn(newColumn); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java index ed8ae7412..210843fca 100644 --- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java @@ -1,15 +1,43 @@ package org.commonmark.internal; -import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; -import org.commonmark.parser.*; -import org.commonmark.parser.block.*; -import org.commonmark.parser.delimiter.DelimiterProcessor; - import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.inline.BlankLineParser; +import org.commonmark.internal.util.Parsing; +import org.commonmark.node.Block; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.Document; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.ListBlock; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SourceSpan; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.IncludeSourceSpans; +import org.commonmark.parser.InlineParser; +import org.commonmark.parser.InlineParserFactory; +import org.commonmark.parser.SourceLine; +import org.commonmark.parser.SourceLines; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockParser; +import org.commonmark.parser.block.BlockParserFactory; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; +import org.commonmark.parser.delimiter.DelimiterProcessor; public class DocumentParser implements ParserState { @@ -275,13 +303,19 @@ private void parseLine(CharSequence ln) { addLine(); } else { - // finalize any blocks not matched if (unmatchedBlocks > 0) { closeBlockParsers(unmatchedBlocks); } if (!blockParser.isContainer()) { + // Capture any blank lines within a separate node for roundtrip purposes + // BlankLine nodes do _not_ interrupt the current node type's processor + if(isBlank()) { + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } + addLine(); } else if (!isBlank()) { // create paragraph container for line @@ -297,6 +331,13 @@ private void parseLine(CharSequence ln) { // // The first line does not start a paragraph yet, but we still want to record source positions. addSourceSpans(); + + // Capture any blank lines within a separate node for roundtrip purposes + // BlankLine nodes do _not_ interrupt the current node type's processor + if(isBlank()) { + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } } } } @@ -403,18 +444,23 @@ private void addLine() { } sb.append(rest); content = sb.toString(); - } else if (index == 0) { - content = line.getContent(); } else { - content = line.getContent().subSequence(index, line.getContent().length()); + content = line.getContent().toString(); } + SourceSpan sourceSpan = null; if (includeSourceSpans == IncludeSourceSpans.BLOCKS_AND_INLINES) { // Note that if we're in a partially-consumed tab, the length here corresponds to the content but not to the // actual source length. That sounds like a problem, but I haven't found a test case where it matters (yet). - sourceSpan = SourceSpan.of(lineIndex, index, content.length()); + sourceSpan = SourceSpan.of(lineIndex, index, line.getLiteralLine().getContent().length()); } - getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan)); + + if(!columnIsInTab) { + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, index)); + }else { + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, 0)); + } + addSourceSpans(); } @@ -425,6 +471,11 @@ private void addSourceSpans() { OpenBlockParser openBlockParser = openBlockParsers.get(i); int blockIndex = openBlockParser.sourceIndex; int length = line.getContent().length() - blockIndex; + + if(line.getLiteralIndex() != 0) { + length = line.getLiteralLine().getContent().length() - blockIndex; + } + if (length != 0) { openBlockParser.blockParser.addSourceSpan(SourceSpan.of(lineIndex, blockIndex, length)); } @@ -481,12 +532,21 @@ private void processInlines() { * its parent, and so on until we find a block that can accept children. */ private void addChild(OpenBlockParser openBlockParser) { - while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { - closeBlockParsers(1); + // BlankLine nodes do _not_ interrupt the current node type's processor, as + // they are non-standard CommonMark nodes meant only for roundtrip processing + if(!(openBlockParser.blockParser instanceof BlankLineParser)) { + while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { + closeBlockParsers(1); + } } getActiveBlockParser().getBlock().appendChild(openBlockParser.blockParser.getBlock()); - activateBlockParser(openBlockParser); + + // BlankLine nodes do _not_ become the active parser, as this would interrupt the + // parser before it + if(!(openBlockParser.blockParser instanceof BlankLineParser)) { + activateBlockParser(openBlockParser); + } } private void activateBlockParser(OpenBlockParser openBlockParser) { @@ -520,6 +580,16 @@ private Block prepareActiveBlockParserForReplacement() { private Document finalizeAndProcess() { closeBlockParsers(openBlockParsers.size()); processInlines(); + Document docNode = documentBlockParser.getBlock(); + String whitespaceEndOfDocument = ""; + + // CommonMark test cases end with newlines, indicating that this should be standard + // in roundtrip scenarios. If a post-document whitespace doesn't exist, add it. + if(whitespaceEndOfDocument.isEmpty()) { + whitespaceEndOfDocument = "\n"; + } + + docNode.setEndOfDocumentWhitespace(whitespaceEndOfDocument); return documentBlockParser.getBlock(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java index 2d7d2c0c9..9f9982592 100644 --- a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java @@ -4,9 +4,12 @@ import org.commonmark.node.Block; import org.commonmark.node.FencedCodeBlock; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import static org.commonmark.internal.util.Escaping.unescapeString; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class FencedCodeBlockParser extends AbstractBlockParser { @@ -14,11 +17,15 @@ public class FencedCodeBlockParser extends AbstractBlockParser { private String firstLine; private StringBuilder otherLines = new StringBuilder(); + private StringBuilder rawLines = new StringBuilder(); public FencedCodeBlockParser(char fenceChar, int fenceLength, int fenceIndent) { block.setFenceChar(fenceChar); - block.setFenceLength(fenceLength); - block.setFenceIndent(fenceIndent); + block.setStartFenceLength(fenceLength); + + // Fenced code blocks can't be indented by newlines or tabs, so it's safe to assume + // spaces for the indentation + block.setPreStartFenceWhitespace(Parsing.generateSpaces(fenceIndent)); } @Override @@ -32,11 +39,20 @@ public BlockContinue tryContinue(ParserState state) { int newIndex = state.getIndex(); CharSequence line = state.getLine().getContent(); if (state.getIndent() < Parsing.CODE_BLOCK_INDENT && nextNonSpace < line.length() && line.charAt(nextNonSpace) == block.getFenceChar() && isClosing(line, nextNonSpace)) { + // Capture whitespace before closing fence (if any) for roundtrip purposes + if(nextNonSpace > 0) { + block.setPreEndFenceWhitespace(Parsing.collectWhitespace(line, 0, nextNonSpace)); + } + // closing fence - we're at end of line, so we can finalize now return BlockContinue.finished(); } else { + // Capture line before optional spaces are removed + rawLines.append(line.toString()); + rawLines.append('\n'); + // skip optional spaces of fence indent - int i = block.getFenceIndent(); + int i = block.getStartFenceIndent(); int length = line.length(); while (i > 0 && newIndex < length && line.charAt(newIndex) == ' ') { newIndex++; @@ -49,9 +65,24 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { if (firstLine == null) { - firstLine = line.getContent().toString(); + + // Raw line is the same as literal line, so just use it + if(line.getLiteralIndex() == 0) { + firstLine = line.getContent().toString(); + } + // Raw line differs from literal line, so preserve literal line + else { + firstLine = line.substring(line.getLiteralIndex(), line.getContent().length()).getContent().toString(); + } } else { - otherLines.append(line.getContent()); + // Raw line is the same as literal line, so just use it + if(line.getLiteralIndex() == 0) { + otherLines.append(line.getContent()); + } + // Raw line differs from literal line, so preserve literal line + else { + otherLines.append(line.getLiteralLine().getContent()); + } otherLines.append('\n'); } } @@ -59,7 +90,22 @@ public void addLine(SourceLine line) { @Override public void closeBlock() { // first line becomes info string - block.setInfo(unescapeString(firstLine.trim())); + // It is not trimmed or escaped here to allow roundtrip processing + block.setInfo(firstLine); + + // Remove final newline in raw content, if it exists + if(rawLines != null && rawLines.length() != 0) { + if(rawLines.charAt(rawLines.length() - 1) == '\n') { + rawLines.deleteCharAt(rawLines.length() - 1); + } + } + + // Capture raw content (no indentation removed) between end of info string + // and beginning of end fence + block.setRaw(rawLines.toString()); + + // Capture content (without indentation) between end of info string + // and beginning of end fence block.setLiteral(otherLines.toString()); } @@ -72,10 +118,14 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar return BlockStart.none(); } + if(indent == 0) { + indent = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()).length(); + } + int nextNonSpace = state.getNextNonSpaceIndex(); FencedCodeBlockParser blockParser = checkOpener(state.getLine().getContent(), nextNonSpace, indent); if (blockParser != null) { - return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getFenceLength()); + return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getStartFenceLength()); } else { return BlockStart.none(); } @@ -120,13 +170,22 @@ private static FencedCodeBlockParser checkOpener(CharSequence line, int index, i // code fence. private boolean isClosing(CharSequence line, int index) { char fenceChar = block.getFenceChar(); - int fenceLength = block.getFenceLength(); + int fenceLength = block.getStartFenceLength(); int fences = Parsing.skip(fenceChar, line, index, line.length()) - index; if (fences < fenceLength) { return false; + }else { + block.setEndFenceLength(fences); } + // spec: The closing code fence [...] may be followed only by spaces, which are ignored. int after = Parsing.skipSpaceTab(line, index + fences, line.length()); + + // Capture post-fence spaces (if any) for roundtrip purposes + if(after > 0 && after != (index + fences)) { + block.setPostBlockWhitespace(Parsing.collectWhitespaceBackwards(line, line.length() - 1, 0)); + } + return after == line.length(); } } diff --git a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java index 81c60d0d1..fd3b642bc 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java @@ -8,7 +8,12 @@ import org.commonmark.parser.InlineParser; import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import org.commonmark.parser.block.*; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class HeadingParser extends AbstractBlockParser { @@ -17,8 +22,19 @@ public class HeadingParser extends AbstractBlockParser { public HeadingParser(int level, SourceLines content) { block.setLevel(level); + block.setSymbolType('#'); this.content = content; } + + public HeadingParser(int level, SourceLines content, char symbolType, int numEndingSymbols, String whitespacePreBlock, String whitespacePreContent, String whitespacePostContent, String whitespacePostBlock) { + this(level, content); + block.setSymbolType(symbolType); + block.setNumEndingSymbol(numEndingSymbols); + block.setPreBlockWhitespace(whitespacePreBlock); + block.setPreContentWhitespace(whitespacePreContent); + block.setPostContentWhitespace(whitespacePostContent); + block.setPostBlockWhitespace(whitespacePostBlock); + } @Override public Block getBlock() { @@ -46,8 +62,24 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar SourceLine line = state.getLine(); int nextNonSpace = state.getNextNonSpaceIndex(); + + String preBlockWhitespace = ""; + if (line.getContent().charAt(nextNonSpace) == '#') { - HeadingParser atxHeading = getAtxHeading(line.substring(nextNonSpace, line.getContent().length())); + + // Account for block-in-a-block instances during roundtrips, like Heading inside a List + if(nextNonSpace > 0) { + preBlockWhitespace = Parsing.collectWhitespaceBackwards(line.getContent(), nextNonSpace - 1, 0); + + // If these values don't match, it's a block-in-a-block scenario because + // there's another character before the whitespace + if(preBlockWhitespace.length() != nextNonSpace) { + preBlockWhitespace = ""; + } + } + + HeadingParser atxHeading = getAtxHeading(line.substring(nextNonSpace - preBlockWhitespace.length(), line.getContent().length()), nextNonSpace); + if (atxHeading != null) { return BlockStart.of(atxHeading).atIndex(line.getContent().length()); } @@ -55,9 +87,42 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar int setextHeadingLevel = getSetextHeadingLevel(line.getContent(), nextNonSpace); if (setextHeadingLevel > 0) { + int numEndingSymbols = line.getContent().toString().trim().length(); + + // AST: Setext headings have a slightly different twist on + // whitespace, see the 
Heading
class for details + String postContentWhitespace = Parsing.collectWhitespace(line.getContent(), 0, line.getContent().length()); + String postBlockWhitespace = Parsing.collectWhitespaceBackwards(line.getContent(), line.getContent().length() - 1, 0); + + // If the setext had valid content, the current matched block + // parser will be the paragraph parser. Setext headings use + // Text (not Paragraph) in the final output, but it's still + // possible to gather the right whitespace from Paragraph. + boolean isMatchedParagraph = matchedBlockParser.getMatchedBlockParser() instanceof ParagraphParser; + + if(isMatchedParagraph) { + SourceLines rawLines = ((ParagraphParser)matchedBlockParser.getMatchedBlockParser()).getRawParagraphLines(); + + // In some instances, the paragraph object exists but has not yet been populated. + // However, the link reference definition parser will still have the paragraph's + // lines ready for capture. If no whitespace is available yet, capture any initial + // whitespace in the first paragraph line. + if(preBlockWhitespace.isEmpty() && !rawLines.isEmpty()) { + CharSequence rawParagraphSequence = rawLines.getLines().get(0).getContent(); + preBlockWhitespace = Parsing.collectWhitespace(rawParagraphSequence.toString(), 0, rawParagraphSequence.length()); + } + } + + char symbolType; + if(setextHeadingLevel == 1) { + symbolType = '='; + }else { + symbolType = '-'; + } + SourceLines paragraph = matchedBlockParser.getParagraphLines(); if (!paragraph.isEmpty()) { - return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph)) + return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph, symbolType, numEndingSymbols, preBlockWhitespace, "", postContentWhitespace, postBlockWhitespace)) .atIndex(line.getContent().length()) .replaceActiveBlockParser(); } @@ -72,7 +137,14 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar // sequence of # characters must be followed by a space or by the end of line. The optional closing sequence of #s // must be preceded by a space and may be followed by spaces only. private static HeadingParser getAtxHeading(SourceLine line) { + return getAtxHeading(line, 0); + } + + private static HeadingParser getAtxHeading(SourceLine line, int nextNonSpace) { + String preBlockWhitespace = ""; + Scanner scanner = Scanner.of(SourceLines.of(line)); + preBlockWhitespace = scanner.whitespaceAsString(); int level = scanner.matchMultiple('#'); if (level == 0 || level > 6) { @@ -89,23 +161,28 @@ private static HeadingParser getAtxHeading(SourceLine line) { return null; } - scanner.whitespace(); + String preContentWhitespace = scanner.whitespaceAsString(); Position start = scanner.position(); Position end = start; boolean hashCanEnd = true; + int numEndingSymbols = 0; + StringBuilder postContentWhitespace = new StringBuilder(); + String postBlockWhitespace = ""; while (scanner.hasNext()) { char c = scanner.peek(); switch (c) { case '#': if (hashCanEnd) { - scanner.matchMultiple('#'); - int whitespace = scanner.whitespace(); + numEndingSymbols = scanner.matchMultiple('#'); + String whitespace = scanner.whitespaceAsString(); // If there's other characters, the hashes and spaces were part of the heading if (scanner.hasNext()) { + numEndingSymbols = 0; + postContentWhitespace.setLength(0); end = scanner.position(); } - hashCanEnd = whitespace > 0; + hashCanEnd = whitespace.length() > 0; } else { scanner.next(); end = scanner.position(); @@ -114,21 +191,29 @@ private static HeadingParser getAtxHeading(SourceLine line) { case ' ': case '\t': hashCanEnd = true; + postContentWhitespace.append(c); scanner.next(); break; default: hashCanEnd = false; + postContentWhitespace.setLength(0); scanner.next(); end = scanner.position(); } } + + if(numEndingSymbols > 0) { + int beforePostBlockWhitespaceIndex = Parsing.skipSpaceTabBackwards(line.getContent(), line.getContent().length() - 1, 0); + postBlockWhitespace = line.getContent().subSequence(beforePostBlockWhitespaceIndex + 1, line.getContent().length()).toString(); + } SourceLines source = scanner.getSource(start, end); String content = source.getContent(); - if (content.isEmpty()) { - return new HeadingParser(level, SourceLines.empty()); + + if(content.isEmpty()) { + return new HeadingParser(level, SourceLines.empty(), '#', numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace.toString(), postBlockWhitespace); } - return new HeadingParser(level, source); + return new HeadingParser(level, source, '#', numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace.toString(), postBlockWhitespace); } // spec: A setext heading underline is a sequence of = characters or a sequence of - characters, with no more than diff --git a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java index 0e2050567..2dc964217 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java @@ -1,13 +1,18 @@ package org.commonmark.internal; +import java.util.regex.Pattern; + import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.HtmlBlock; import org.commonmark.node.Paragraph; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import java.util.regex.Pattern; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class HtmlBlockParser extends AbstractBlockParser { @@ -64,6 +69,7 @@ public class HtmlBlockParser extends AbstractBlockParser { private boolean finished = false; private BlockContent content = new BlockContent(); + private BlockContent rawContent = new BlockContent(); private HtmlBlockParser(Pattern closingPattern) { this.closingPattern = closingPattern; @@ -90,7 +96,22 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - content.add(line.getContent()); + // Raw line and literal line are the same + if(line.getLiteralIndex() == 0) { + content.add(line.getContent()); + rawContent.add(line.getContent()); + } + // Raw line and literal line are different + else { + content.add(line.getLiteralLine().getContent()); + + // Use the literal line for anything which may be a block quote + if(line != null && line.getContent().toString().contains(">")) { + rawContent.add(line.getContent()); + }else { + rawContent.add(line.getLiteralLine().getContent()); + } + } if (closingPattern != null && closingPattern.matcher(line.getContent()).find()) { finished = true; @@ -100,7 +121,9 @@ public void addLine(SourceLine line) { @Override public void closeBlock() { block.setLiteral(content.getString()); + block.setRaw(rawContent.getString()); content = null; + rawContent = null; } public static class Factory extends AbstractBlockParserFactory { diff --git a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java index af74a587c..65f34e665 100644 --- a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java @@ -1,20 +1,35 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.List; + import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.IndentedCodeBlock; import org.commonmark.node.Paragraph; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import java.util.ArrayList; -import java.util.List; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class IndentedCodeBlockParser extends AbstractBlockParser { private final IndentedCodeBlock block = new IndentedCodeBlock(); private final List lines = new ArrayList<>(); + private final List rawLines = new ArrayList<>(); + // Preserve original default constructor by explicitly defining one + public IndentedCodeBlockParser() { + super(); + } + + public IndentedCodeBlockParser(String indentWhitespace) { + block.setIndentWhitespace(indentWhitespace); + } + @Override public Block getBlock() { return block; @@ -33,6 +48,22 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { + String prefix = ""; + + // Capture the prefix (needed for roundtrip, but unnecessary for HTML processing) + if(line.getLiteralIndex() != 0) { + prefix = line.substring(0, line.getLiteralIndex()).getContent().toString(); + line = line.getLiteralLine(); + } + + // Strip leading whitespace off first raw line (because it's already + // captured by the whitespace tracker) + if(rawLines != null && rawLines.size() == 0) { + rawLines.add(line.getContent().toString().replaceFirst("^\\s+", "")); + }else { + rawLines.add(prefix + line.getContent()); + } + lines.add(line.getContent()); } @@ -52,8 +83,28 @@ public void closeBlock() { sb.append('\n'); } + // The "literal" string is optimized for HTML formatting, but omits some details + // which are important for roundtrip rendering String literal = sb.toString(); + + // Reuse the existing StringBuilder as a small optimization + sb.setLength(0); + for (int i = 0; i < rawLines.size(); i++) { + sb.append(rawLines.get(i)); + + // Don't append a newline if this block only contains one line, or to the + // last line if there is more than one line + if(rawLines.size() != 1 && i != rawLines.size() - 1) { + sb.append('\n'); + } + } + + // The "raw" string is, as much as possible, the entire raw content as it was first + // entered into the parser. This allows roundtrip parsing for indented code blocks. + String raw = sb.toString(); + block.setLiteral(literal); + block.setRaw(raw); } public static class Factory extends AbstractBlockParserFactory { @@ -62,7 +113,9 @@ public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { // An indented code block cannot interrupt a paragraph. if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT && !state.isBlank() && !(state.getActiveBlockParser().getBlock() instanceof Paragraph)) { - return BlockStart.of(new IndentedCodeBlockParser()).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); + String indentWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()); + + return BlockStart.of(new IndentedCodeBlockParser(indentWhitespace)).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index c14b9e885..c9904f9d3 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -1,18 +1,44 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.inline.AsteriskDelimiterProcessor; +import org.commonmark.internal.inline.AutolinkInlineParser; +import org.commonmark.internal.inline.BackslashInlineParser; +import org.commonmark.internal.inline.BackticksInlineParser; +import org.commonmark.internal.inline.EntityInlineParser; +import org.commonmark.internal.inline.HtmlInlineParser; +import org.commonmark.internal.inline.InlineContentParser; +import org.commonmark.internal.inline.InlineParserState; +import org.commonmark.internal.inline.ParsedInline; +import org.commonmark.internal.inline.ParsedInlineImpl; +import org.commonmark.internal.inline.Position; import org.commonmark.internal.inline.Scanner; -import org.commonmark.internal.inline.*; +import org.commonmark.internal.inline.UnderscoreDelimiterProcessor; import org.commonmark.internal.util.Escaping; import org.commonmark.internal.util.LinkScanner; import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Image; +import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.Node; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.SourceSpans; +import org.commonmark.node.Text; import org.commonmark.parser.InlineParser; import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.SourceLines; import org.commonmark.parser.delimiter.DelimiterProcessor; -import java.util.*; - public class InlineParserImpl implements InlineParser, InlineParserState { private final BitSet specialCharacters; @@ -126,7 +152,7 @@ public void parse(SourceLines lines, Node block) { } } - processDelimiters(null); + processDelimiters(null, ""); mergeChildTextNodes(block); } @@ -138,8 +164,16 @@ void reset(SourceLines lines) { this.lastBracket = null; } + // Text node without roundtrip information private Text text(SourceLines sourceLines) { - Text text = new Text(sourceLines.getContent()); + Text text = new Text(sourceLines.getContent(), sourceLines.getContent(), "", ""); + text.setSourceSpans(sourceLines.getSourceSpans()); + return text; + } + + // Text node with roundtrip information + private Text text(SourceLines sourceLines, String preContentWhitespace, String postContentWhitespace) { + Text text = new Text(sourceLines.getContent(), sourceLines.getContent(), preContentWhitespace, postContentWhitespace); text.setSourceSpans(sourceLines.getSourceSpans()); return text; } @@ -150,6 +184,9 @@ private Text text(SourceLines sourceLines) { * On failure, return null. */ private List parseInline() { + // AST: Capture raw information needed for roundtrip rendering + String prefix = scanner.alignToLiteral(); + char c = scanner.peek(); switch (c) { @@ -158,7 +195,9 @@ private List parseInline() { case '!': return Collections.singletonList(parseBang()); case ']': - return Collections.singletonList(parseCloseBracket()); + List nodeList = Collections.singletonList(parseCloseBracket(prefix)); + prefix = ""; + return nodeList; case '\n': return Collections.singletonList(parseLineBreak()); case Scanner.END: @@ -167,14 +206,23 @@ private List parseInline() { // No inline parser, delimiter or other special handling. if (!specialCharacters.get(c)) { - return Collections.singletonList(parseText()); + List nodeList = Collections.singletonList(parseText(prefix)); + prefix = ""; + return nodeList; } List inlineParsers = this.inlineParsers.get(c); if (inlineParsers != null) { Position position = scanner.position(); for (InlineContentParser inlineParser : inlineParsers) { - ParsedInline parsedInline = inlineParser.tryParse(this); + ParsedInline parsedInline; + if(inlineParser instanceof BackticksInlineParser) { + parsedInline = inlineParser.tryParse(this, prefix); + prefix = ""; + }else { + parsedInline = inlineParser.tryParse(this); + } + if (parsedInline instanceof ParsedInlineImpl) { ParsedInlineImpl parsedInlineImpl = (ParsedInlineImpl) parsedInline; Node node = parsedInlineImpl.getNode(); @@ -192,21 +240,28 @@ private List parseInline() { DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c); if (delimiterProcessor != null) { - List nodes = parseDelimiters(delimiterProcessor, c); + List nodes = parseDelimiters(delimiterProcessor, c, prefix); + + // Preserve a prefix which has _only_ whitespace, as this may be + // needed in the following "parseText" method + if(!prefix.isEmpty() && prefix.trim().equals("")) { + prefix = ""; + } + if (nodes != null) { return nodes; } } // If we get here, even for a special/delimiter character, we will just treat it as text. - return Collections.singletonList(parseText()); + return Collections.singletonList(parseText(prefix)); } /** * Attempt to parse delimiters like emphasis, strong emphasis or custom delimiters. */ - private List parseDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar) { - DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar); + private List parseDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar, String prefix) { + DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar, prefix); if (res == null) { return null; } @@ -245,6 +300,7 @@ private Node parseOpenBracket() { private Node parseBang() { Position start = scanner.position(); scanner.next(); + if (scanner.next('[')) { Position contentPosition = scanner.position(); Text node = text(scanner.getSource(start, contentPosition)); @@ -262,8 +318,13 @@ private Node parseBang() { * plain [ character. If there is a matching delimiter, remove it from the delimiter stack. */ private Node parseCloseBracket() { + return parseCloseBracket(""); + } + + private Node parseCloseBracket(String prefix) { Position beforeClose = scanner.position(); scanner.next(); + Position afterClose = scanner.position(); // Get previous `[` or `![` @@ -281,31 +342,67 @@ private Node parseCloseBracket() { // Check to see if we have a link/image String dest = null; + String rawDest = ""; String title = null; + String rawTitle = ""; + + LinkType currentLinkType = LinkType.NULL; + char titleSymbol = Character.MIN_VALUE; + String whitespacePreDestination = ""; + String whitespacePreTitle = ""; + String whitespacePostContent = ""; // Maybe a inline link like `[foo](/uri "title")` if (scanner.next('(')) { + whitespacePreDestination = scanner.whitespaceAsString(); scanner.whitespace(); + rawDest = parseLinkDestinationRaw(scanner); dest = parseLinkDestination(scanner); + if (dest == null) { scanner.setPosition(afterClose); + whitespacePreDestination = ""; } else { - int whitespace = scanner.whitespace(); + whitespacePreTitle = scanner.whitespaceAsString(); + // title needs a whitespace before - if (whitespace >= 1) { + if (whitespacePreTitle.length() >= 1) { + Position titleStart = scanner.position(); + // Capture the symbol used for the title + titleSymbol = scanner.peek(); + rawTitle = parseLinkTitleRaw(scanner); + scanner.setPosition(titleStart); title = parseLinkTitle(scanner); - scanner.whitespace(); + + // If title isn't valid, discard the captured value of its symbol + if(title == null) { + titleSymbol = Character.MIN_VALUE; + } + whitespacePostContent = scanner.whitespaceAsString(); } + + currentLinkType = LinkType.INLINE; + if (!scanner.next(')')) { // Don't have a closing `)`, so it's not a destination and title -> reset. // Note that something like `[foo](` could be valid, `(` will just be text. scanner.setPosition(afterClose); dest = null; + rawDest = ""; title = null; + rawTitle = ""; + currentLinkType = LinkType.NULL; + titleSymbol = Character.MIN_VALUE; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; } } } + String label = null; + String rawLabel = null; + // Maybe a reference link like `[foo][bar]`, `[foo][]` or `[foo]`. // Note that even `[foo](` could be a valid link if there's a reference, which is why this is not just an `else` // here. @@ -327,13 +424,55 @@ private Node parseCloseBracket() { if (definition != null) { dest = definition.getDestination(); title = definition.getTitle(); + label = definition.getLabel(); + }else { + if(!ref.isEmpty()) { + label = ref; + } } + + currentLinkType = LinkType.REFERENCE; + + // If this reference link has a following label (like + // `[foo][bar]` or `[foo][]`, capture it for roundtrip + // purposes + Position currentPos = scanner.position(); + + scanner.setPosition(beforeClose); + scanner.next(); + + whitespacePostContent = scanner.whitespaceAsString(); + + // Check just in case there's a "[]" hanging out, but if + // there are any line breaks ignore extra symbols + // because a next pass would catch those. + if(scanner.hasNext() && scanner.peek() == '[' && !whitespacePostContent.contains("\n")) { + scanner.next(); + if(scanner.peek() == ']') { + label = ""; + } + }else { + label = null; + } + + // The raw label needs to be populated, either with the original (raw) + // link reference info or with the (literal) label information + if(label != null && !label.isEmpty() && !ref.equals(label)) { + rawLabel = ref; + }else { + rawLabel = label; + } + + // Reset scanner if no extra content was found + scanner.setPosition(currentPos); } } if (dest != null) { // If we got here, we have a link or image - Node linkOrImage = opener.image ? new Image(dest, title) : new Link(dest, title); + Node linkOrImage = opener.image ? + new Image(dest, rawDest, title, rawTitle, label, rawLabel, currentLinkType, titleSymbol, whitespacePreDestination, whitespacePreTitle, whitespacePostContent) : + new Link(dest, rawDest, title, rawTitle, label, rawLabel, currentLinkType, titleSymbol, whitespacePreDestination, whitespacePreTitle, whitespacePostContent); // Add all nodes between the opening bracket and now (closing bracket) as child nodes of the link Node node = opener.node.getNext(); @@ -348,7 +487,7 @@ private Node parseCloseBracket() { } // Process delimiters such as emphasis inside link/image - processDelimiters(opener.previousDelimiter); + processDelimiters(opener.previousDelimiter, prefix); mergeChildTextNodes(linkOrImage); // We don't need the corresponding text node anymore, we turned it into a link/image node opener.node.unlink(); @@ -373,7 +512,7 @@ private Node parseCloseBracket() { removeLastBracket(); scanner.setPosition(afterClose); - return text(scanner.getSource(beforeClose, afterClose)); + return text(scanner.getSource(beforeClose, afterClose), prefix, ""); } } @@ -409,6 +548,18 @@ private String parseLinkDestination(Scanner scanner) { return Escaping.unescapeString(dest); } + + private String parseLinkDestinationRaw(Scanner rawScanner) { + Position start = rawScanner.position(); + if(!LinkScanner.scanLinkDestination(rawScanner)) { + return null; + } + + String dest = rawScanner.getSource(start, scanner.position()).getContent(); + rawScanner.setPosition(start); + + return dest; + } /** * Attempt to parse link title (sans quotes), returning the string or null if no match. @@ -424,6 +575,20 @@ private String parseLinkTitle(Scanner scanner) { String title = rawTitle.substring(1, rawTitle.length() - 1); return Escaping.unescapeString(title); } + + /** + * Attempt to parse the link title in raw form, returning the raw string or null if no match. + */ + private String parseLinkTitleRaw(Scanner rawScanner) { + Position start = rawScanner.position(); + if(!LinkScanner.scanLinkTitle(rawScanner)) { + return null; + } + + String rawTitle = scanner.getSource(start, scanner.position()).getContent(); + + return rawTitle; + } /** * Attempt to parse a link label, returning the label between the brackets or null. @@ -466,32 +631,69 @@ private Node parseLineBreak() { * Parse the next character as plain text, and possibly more if the following characters are non-special. */ private Node parseText() { + return parseText(""); + } + + private Node parseText(String prefix) { + String preContentWhitespace = ""; + + if(!prefix.isEmpty()) { + // Capture whitespace and anything that might be a block quote for roundtrip purposes + if(prefix.trim().equals("") || prefix.contains(">")) { + preContentWhitespace = prefix; + } + } + + StringBuilder postContentWhitespace = new StringBuilder(); + Position start = scanner.position(); scanner.next(); + char c; while (true) { c = scanner.peek(); - if (c == Scanner.END || specialCharacters.get(c)) { + + if(c == Scanner.END) { break; } + + if(c != ' ' && c != '\t' && c != '\n' && postContentWhitespace.length() > 0) { + postContentWhitespace.setLength(0); + } + + if (specialCharacters.get(c)) { + break; + } + + if(c == ' ' || c == '\t') { + postContentWhitespace.append(c); + } + scanner.next(); } - - SourceLines source = scanner.getSource(start, scanner.position()); + + SourceLines source = scanner.getSource(start, scanner.position(), prefix.length()); String content = source.getContent(); if (c == '\n') { // We parsed until the end of the line. Trim any trailing spaces and remember them (for hard line breaks). int end = Parsing.skipBackwards(' ', content, content.length() - 1, 0) + 1; trailingSpaces = content.length() - end; + if(postContentWhitespace.length() == 0) { + postContentWhitespace.append(content.subSequence(end, content.length())); + } content = content.substring(0, end); } else if (c == Scanner.END) { // For the last line, both tabs and spaces are trimmed for some reason (checked with commonmark.js). int end = Parsing.skipSpaceTabBackwards(content, content.length() - 1, 0) + 1; + if(postContentWhitespace.length() == 0) { + postContentWhitespace.append(content.subSequence(end, content.length())); + } content = content.substring(0, end); } - - Text text = new Text(content); + + Text text = new Text(content, content, preContentWhitespace, postContentWhitespace.toString()); + text.setSourceSpans(source.getSourceSpans()); return text; } @@ -502,7 +704,7 @@ private Node parseText() { * * @return information about delimiter run, or {@code null} */ - private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar) { + private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar, String prefix) { int before = scanner.peekPreviousCodePoint(); Position start = scanner.position(); @@ -517,8 +719,16 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char List delimiters = new ArrayList<>(); scanner.setPosition(start); Position positionBefore = start; + while (scanner.next(delimiterChar)) { - delimiters.add(text(scanner.getSource(positionBefore, scanner.position()))); + Text delimText = text(scanner.getSource(positionBefore, scanner.position()), prefix, ""); + + // Erase prefix after first line + if(prefix.length() > 0) { + prefix = ""; + } + + delimiters.add(delimText); positionBefore = scanner.position(); } @@ -548,6 +758,10 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char } private void processDelimiters(Delimiter stackBottom) { + processDelimiters(stackBottom, ""); + } + + private void processDelimiters(Delimiter stackBottom, String prefix) { Map openersBottom = new HashMap<>(); @@ -576,7 +790,13 @@ private void processDelimiters(Delimiter stackBottom) { while (opener != null && opener != stackBottom && opener != openersBottom.get(delimiterChar)) { if (opener.canOpen() && opener.delimiterChar == openingDelimiterChar) { potentialOpenerFound = true; - usedDelims = delimiterProcessor.process(opener, closer); + usedDelims = delimiterProcessor.process(opener, closer, prefix); + + String openerWhitespace = opener.characters.get(0).whitespacePreContent(); + if(!openerWhitespace.trim().equals("")) { + closer.characters.get(0).setWhitespace(openerWhitespace, ""); + } + if (usedDelims > 0) { openerFound = true; break; @@ -713,7 +933,9 @@ private void mergeTextNodesInclusive(Node fromNode, Node toNode) { private void mergeIfNeeded(Text first, Text last, int textLength) { if (first != null && last != null && first != last) { StringBuilder sb = new StringBuilder(textLength); + StringBuilder sb2 = new StringBuilder(textLength); sb.append(first.getLiteral()); + sb2.append(first.getRaw()); SourceSpans sourceSpans = null; if (includeSourceSpans) { sourceSpans = new SourceSpans(); @@ -723,6 +945,7 @@ private void mergeIfNeeded(Text first, Text last, int textLength) { Node stop = last.getNext(); while (node != stop) { sb.append(((Text) node).getLiteral()); + sb2.append(((Text)node).getRaw()); if (sourceSpans != null) { sourceSpans.addAll(node.getSourceSpans()); } @@ -732,7 +955,9 @@ private void mergeIfNeeded(Text first, Text last, int textLength) { unlink.unlink(); } String literal = sb.toString(); + String raw = sb2.toString(); first.setLiteral(literal); + first.setRaw(raw); if (sourceSpans != null) { first.setSourceSpans(sourceSpans.getSourceSpans()); } diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java index 2bceb7549..8a8a23811 100644 --- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java @@ -1,5 +1,8 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.List; + import org.commonmark.internal.inline.Position; import org.commonmark.internal.inline.Scanner; import org.commonmark.internal.util.Escaping; @@ -9,9 +12,6 @@ import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import java.util.ArrayList; -import java.util.List; - /** * Parser for link reference definitions at the beginning of a paragraph. * @@ -26,13 +26,24 @@ public class LinkReferenceDefinitionParser { private final List sourceSpans = new ArrayList<>(); private StringBuilder label; + private String extendedLabel = null; private String destination; + private String rawDestination; private char titleDelimiter; private StringBuilder title; private boolean referenceValid = false; + private List rawParagraphLines = new ArrayList<>(); + + private int previousLineIndex = 0; + private String whitespacePreLabel = ""; + private String whitespacePreDestination = ""; + private String whitespacePreTitle = ""; + private String whitespacePostTitle = ""; public void parse(SourceLine line) { - paragraphLines.add(line); + paragraphLines.add(line.getLiteralLine()); + rawParagraphLines.add(line); + if (state == State.PARAGRAPH) { // We're in a paragraph now. Link reference definitions can only appear at the beginning, so once // we're in a paragraph, there's no going back. @@ -40,6 +51,7 @@ public void parse(SourceLine line) { } Scanner scanner = Scanner.of(SourceLines.of(line)); + while (scanner.hasNext()) { boolean success; switch (state) { @@ -48,14 +60,35 @@ public void parse(SourceLine line) { break; } case LABEL: { + // Capture line number to determine if line breaks occur + if(line.getSourceSpan() != null) { + previousLineIndex = line.getSourceSpan().getLineIndex(); + } + success = label(scanner); break; } case DESTINATION: { + // Check for any line breaks + if(line.getSourceSpan() != null) { + if(line.getSourceSpan().getLineIndex() > previousLineIndex) { + whitespacePreDestination = whitespacePreDestination + "\n"; + previousLineIndex++; + } + } + success = destination(scanner); break; } case START_TITLE: { + // Check for any line breaks + if(line.getSourceSpan() != null) { + if(line.getSourceSpan().getLineIndex() > previousLineIndex) { + whitespacePreTitle = whitespacePreTitle + "\n"; + previousLineIndex++; + } + } + success = startTitle(scanner); break; } @@ -85,10 +118,22 @@ public void addSourceSpan(SourceSpan sourceSpan) { SourceLines getParagraphLines() { return SourceLines.of(paragraphLines); } + + SourceLines getRawParagraphLines() { + return SourceLines.of(rawParagraphLines); + } + + void setRawParagraphLines(SourceLines rawParagraphLines) { + this.rawParagraphLines = rawParagraphLines.getLines(); + } List getParagraphSourceSpans() { return sourceSpans; } + + String getExtendedLabel() { + return extendedLabel; + } List getDefinitions() { finishReference(); @@ -100,7 +145,15 @@ State getState() { } private boolean startDefinition(Scanner scanner) { - scanner.whitespace(); + // Capture whitespace for roundtrip purposes + whitespacePreLabel = scanner.whitespaceAsString(); + + String whitespaceCheck = scanner.alignToLiteral(); + + if(!whitespaceCheck.isEmpty()) { + whitespacePreLabel = whitespaceCheck; + } + if (!scanner.next('[')) { return false; } @@ -144,7 +197,7 @@ private boolean label(Scanner scanner) { state = State.DESTINATION; - scanner.whitespace(); + whitespacePreDestination = scanner.whitespaceAsString(); return true; } else { return false; @@ -152,24 +205,26 @@ private boolean label(Scanner scanner) { } private boolean destination(Scanner scanner) { - scanner.whitespace(); + whitespacePreDestination = whitespacePreDestination + + scanner.whitespaceAsString(); Position start = scanner.position(); if (!LinkScanner.scanLinkDestination(scanner)) { return false; } - String rawDestination = scanner.getSource(start, scanner.position()).getContent(); + rawDestination = scanner.getSource(start, scanner.position()).getContent(); destination = rawDestination.startsWith("<") ? rawDestination.substring(1, rawDestination.length() - 1) : rawDestination; - int whitespace = scanner.whitespace(); + whitespacePreTitle = scanner.whitespaceAsString(); if (!scanner.hasNext()) { // Destination was at end of line, so this is a valid reference for sure (and maybe a title). // If not at end of line, wait for title to be valid first. referenceValid = true; paragraphLines.clear(); - } else if (whitespace == 0) { + rawParagraphLines.clear(); + } else if (whitespacePreTitle.length() == 0) { // spec: The title must be separated from the link destination by whitespace return false; } @@ -179,7 +234,7 @@ private boolean destination(Scanner scanner) { } private boolean startTitle(Scanner scanner) { - scanner.whitespace(); + whitespacePreTitle = whitespacePreTitle + scanner.whitespaceAsString(); if (!scanner.hasNext()) { state = State.START_DEFINITION; return true; @@ -229,14 +284,17 @@ private boolean title(Scanner scanner) { // Skip delimiter character scanner.next(); - scanner.whitespace(); + whitespacePostTitle = scanner.whitespaceAsString(); if (scanner.hasNext()) { + title = null; + whitespacePostTitle = ""; // spec: No further non-whitespace characters may occur on the line. return false; } referenceValid = true; finishReference(); paragraphLines.clear(); + rawParagraphLines.clear(); // See if there's another definition. state = State.START_DEFINITION; @@ -248,9 +306,14 @@ private void finishReference() { return; } + if(rawDestination.isEmpty()) { + rawDestination = destination; + } + String d = Escaping.unescapeString(destination); String t = title != null ? Escaping.unescapeString(title.toString()) : null; - LinkReferenceDefinition definition = new LinkReferenceDefinition(label.toString(), d, t); + String rawTitle = title != null ? title.toString() : ""; + LinkReferenceDefinition definition = new LinkReferenceDefinition(label.toString(), d, rawDestination, t, rawTitle, titleDelimiter, whitespacePreLabel, whitespacePreDestination, whitespacePreTitle, whitespacePostTitle); definition.setSourceSpans(sourceSpans); sourceSpans.clear(); definitions.add(definition); @@ -259,6 +322,8 @@ private void finishReference() { referenceValid = false; destination = null; title = null; + whitespacePreDestination = ""; + whitespacePostTitle = ""; } enum State { diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java index f6702518b..202434ceb 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java @@ -1,8 +1,18 @@ package org.commonmark.internal; import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; -import org.commonmark.parser.block.*; +import org.commonmark.node.Block; +import org.commonmark.node.BulletList; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.OrderedList; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockParser; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class ListBlockParser extends AbstractBlockParser { @@ -11,6 +21,17 @@ public class ListBlockParser extends AbstractBlockParser { private boolean hadBlankLine; private int linesAfterBlank; + private static String currentRawNumber; + + /* + * When gathering data for roundtrip parsing, there is no easy way to know whether + * a blank list item is a precursor to more content (CommonMark test case 248) or + * if it stands alone (CommonMark test case 252). + * Luckily, if the first line is blank, any further content which doesn't start a + * new list item indicates that the first line is split, not standalone. + */ + private static boolean firstLineBlank = false; + public ListBlockParser(ListBlock block) { this.block = block; } @@ -115,8 +136,28 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { case '+': case '*': if (isSpaceTabOrEnd(line, index + 1)) { + // Collect any pre-content whitespace in the first line for + // roundtrip purposes. + String preContentWhitespace = Parsing.collectWhitespace(line, index + 1, line.length()); + + // AST: Lists can be separated from their delimiter by a blank line, make + // sure to capture this for roundtrip purposes if it occurs + if(line.subSequence(index + 1, line.length()).toString().trim().equals("")) { + firstLineBlank = true; + }else { + firstLineBlank = false; + } + + String preBlockWhitespace = line.subSequence(0, index).toString(); + + if(!preBlockWhitespace.trim().equals("")) { + preBlockWhitespace = ""; + } + BulletList bulletList = new BulletList(); bulletList.setBulletMarker(c); + bulletList.setPreBlockWhitespace(preBlockWhitespace); + bulletList.setPreContentWhitespace(preContentWhitespace); return new ListMarkerData(bulletList, index + 1); } else { return null; @@ -153,9 +194,32 @@ private static ListMarkerData parseOrderedList(CharSequence line, int index) { case ')': if (digits >= 1 && isSpaceTabOrEnd(line, i + 1)) { String number = line.subSequence(index, i).toString(); + + // AST: Lists can be separated from their delimiter by a blank line, make + // sure to capture this for roundtrip purposes if it occurs + if(line.subSequence(index + 1, line.length()).toString().trim().equals("")) { + firstLineBlank = true; + }else { + firstLineBlank = false; + } + + // Collect any pre-content whitespace in the first line for + // roundtrip purposes. + String preContentWhitespace = Parsing.collectWhitespace(line, i+1, line.length()); + String preBlockWhitespace = line.subSequence(0, index).toString(); + + if(!preBlockWhitespace.trim().equals("")) { + preBlockWhitespace = ""; + } + OrderedList orderedList = new OrderedList(); orderedList.setStartNumber(Integer.parseInt(number)); + // AST: Actual list number can be anything, it is + // only auto-incremented during rendering + currentRawNumber = number; orderedList.setDelimiter(c); + orderedList.setPreBlockWhitespace(preBlockWhitespace); + orderedList.setPreContentWhitespace(preContentWhitespace); return new ListMarkerData(orderedList, i + 1); } else { return null; @@ -217,8 +281,12 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar } int newColumn = listData.contentColumn; - ListItemParser listItemParser = new ListItemParser(newColumn - state.getColumn()); - + String preBlockWhitespace = listData.listBlock.whitespacePreBlock(); + String preContentWhitespace = listData.listBlock.whitespacePreContent(); + + ListItemParser listItemParser = new ListItemParser(newColumn - state.getColumn(), currentRawNumber, firstLineBlank, preBlockWhitespace, preContentWhitespace); + firstLineBlank = false; + // prepend the list block if needed if (!(matched instanceof ListBlockParser) || !(listsMatch((ListBlock) matched.getBlock(), listData.listBlock))) { diff --git a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java index 6f03770b3..389031173 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java @@ -1,5 +1,6 @@ package org.commonmark.internal; +import org.commonmark.node.BlankLine; import org.commonmark.node.Block; import org.commonmark.node.ListBlock; import org.commonmark.node.ListItem; @@ -20,8 +21,20 @@ public class ListItemParser extends AbstractBlockParser { private boolean hadBlankLine; + // Helps distinguish multi-line line items, as described in 
ListBlockParser
+ private boolean firstLineBlank = false; + public ListItemParser(int contentIndent) { this.contentIndent = contentIndent; + block.setRawNumber(""); + } + + public ListItemParser(int contentIndent, String currentRawNumber, boolean firstLineBlank, String whitespacePreMarker, String whitespacePostMarker) { + this(contentIndent); + block.setRawNumber(currentRawNumber); + this.firstLineBlank = firstLineBlank; + block.setPreMarkerWhitespace(whitespacePreMarker); + block.setPostMarkerWhitespace(whitespacePostMarker); } @Override @@ -52,18 +65,24 @@ public Block getBlock() { @Override public BlockContinue tryContinue(ParserState state) { if (state.isBlank()) { - if (block.getFirstChild() == null) { + if (block.getFirstChild() == null || block.getFirstChild() instanceof BlankLine) { // Blank line after empty list item return BlockContinue.none(); } else { Block activeBlock = state.getActiveBlockParser().getBlock(); // If the active block is a code block, blank lines in it should not affect if the list is tight. hadBlankLine = activeBlock instanceof Paragraph || activeBlock instanceof ListItem; + return BlockContinue.atIndex(state.getNextNonSpaceIndex()); } } if (state.getIndent() >= contentIndent) { + if(firstLineBlank) { + String whitespacePostMarker = block.whitespacePostMarker() + "\n"; + block.setPostMarkerWhitespace(whitespacePostMarker); + firstLineBlank = false; + } return BlockContinue.atColumn(state.getColumn() + contentIndent); } else { // Note: We'll hit this case for lazy continuation lines, they will get added later. diff --git a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java index 89328ef2a..7a39a6c40 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java @@ -1,5 +1,7 @@ package org.commonmark.internal; +import java.util.List; + import org.commonmark.node.Block; import org.commonmark.node.LinkReferenceDefinition; import org.commonmark.node.Paragraph; @@ -11,13 +13,20 @@ import org.commonmark.parser.block.BlockContinue; import org.commonmark.parser.block.ParserState; -import java.util.List; - public class ParagraphParser extends AbstractBlockParser { private final Paragraph block = new Paragraph(); private final LinkReferenceDefinitionParser linkReferenceDefinitionParser = new LinkReferenceDefinitionParser(); + // Preserve original default constructor by explicitly defining one + public ParagraphParser() { + super(); + } + + public ParagraphParser(String preBlockWhitespace) { + block.setPreBlockWhitespace(preBlockWhitespace); + } + @Override public boolean canHaveLazyContinuationLines() { return true; @@ -62,13 +71,21 @@ public void closeBlock() { public void parseInlines(InlineParser inlineParser) { SourceLines lines = linkReferenceDefinitionParser.getParagraphLines(); if (!lines.isEmpty()) { - inlineParser.parse(lines, block); + inlineParser.parse(getRawParagraphLines(), block); } } public SourceLines getParagraphLines() { return linkReferenceDefinitionParser.getParagraphLines(); } + + public SourceLines getRawParagraphLines() { + return linkReferenceDefinitionParser.getRawParagraphLines(); + } + + public void setRawParagraphLines(SourceLines rawParagraphLines) { + linkReferenceDefinitionParser.setRawParagraphLines(rawParagraphLines); + } public List getDefinitions() { return linkReferenceDefinitionParser.getDefinitions(); diff --git a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java index 0fe8065bb..13f9b70b8 100644 --- a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java @@ -73,4 +73,10 @@ private DelimiterProcessor findProcessor(int len) { public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return findProcessor(openingRun.length()).process(openingRun, closingRun); } + + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java index 88ad41ac9..f1c21e83f 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java @@ -2,12 +2,26 @@ import org.commonmark.node.Block; import org.commonmark.node.ThematicBreak; -import org.commonmark.parser.block.*; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class ThematicBreakParser extends AbstractBlockParser { private final ThematicBreak block = new ThematicBreak(); + // Preserve original default constructor by explicitly defining one + public ThematicBreakParser() { + super(); + } + + public ThematicBreakParser(CharSequence content) { + block.setContent(content); + } + @Override public Block getBlock() { return block; @@ -28,8 +42,9 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar } int nextNonSpace = state.getNextNonSpaceIndex(); CharSequence line = state.getLine().getContent(); + if (isThematicBreak(line, nextNonSpace)) { - return BlockStart.of(new ThematicBreakParser()).atIndex(line.length()); + return BlockStart.of(new ThematicBreakParser(line)).atIndex(line.length()); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java index ecfd2d972..ee38b5437 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java @@ -1,11 +1,12 @@ package org.commonmark.internal.inline; +import java.util.regex.Pattern; + import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; import org.commonmark.node.Text; import org.commonmark.parser.SourceLines; -import java.util.regex.Pattern; - /** * Attempt to parse an autolink (URL or email in pointy brackets). */ @@ -39,9 +40,16 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { Text text = new Text(content); text.setSourceSpans(textSource.getSourceSpans()); link.appendChild(text); + link.setLinkType(LinkType.AUTOLINK); return ParsedInline.of(link, scanner.position()); } } return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Autolinks do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java index f57a67a74..8e6d83853 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java @@ -24,12 +24,18 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { char next = scanner.peek(); if (next == '\n') { scanner.next(); - return ParsedInline.of(new HardLineBreak(), scanner.position()); + return ParsedInline.of(new HardLineBreak(true), scanner.position()); } else if (ESCAPABLE.matcher(String.valueOf(next)).matches()) { scanner.next(); - return ParsedInline.of(new Text(String.valueOf(next)), scanner.position()); + return ParsedInline.of(new Text(String.valueOf(next), "\\" + String.valueOf(next), "", ""), scanner.position()); } else { return ParsedInline.of(new Text("\\"), scanner.position()); } } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Backslash-escaped characters do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java index ad079444a..9da813048 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java @@ -14,6 +14,13 @@ public class BackticksInlineParser implements InlineContentParser { public ParsedInline tryParse(InlineParserState inlineParserState) { Scanner scanner = inlineParserState.scanner(); Position start = scanner.position(); + String preBlockWhitespace = ""; + + char listCheck = scanner.peek(); + if(listCheck == '-' || listCheck == '*' || listCheck == '+') { + preBlockWhitespace = listCheck + scanner.whitespaceAsString(); + } + int openingTicks = scanner.matchMultiple('`'); Position afterOpening = scanner.position(); @@ -23,7 +30,11 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { if (count == openingTicks) { Code node = new Code(); + node.setNumBackticks(count); + String content = scanner.getSource(afterOpening, beforeClosing).getContent(); + // Capture raw string for roundtrip rendering + node.setRaw(content); content = content.replace('\n', ' '); // spec: If the resulting string both begins and ends with a space character, but does not consist @@ -42,7 +53,13 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { // If we got here, we didn't find a matching closing backtick sequence. SourceLines source = scanner.getSource(start, afterOpening); - Text text = new Text(source.getContent()); + Text text = new Text(source.getContent(), source.getContent(), preBlockWhitespace, ""); return ParsedInline.of(text, afterOpening); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Backticks do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java new file mode 100644 index 000000000..e50e00c11 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java @@ -0,0 +1,28 @@ +package org.commonmark.internal.inline; + +import org.commonmark.node.BlankLine; +import org.commonmark.node.Block; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.ParserState; + +public class BlankLineParser extends AbstractBlockParser { + + private BlankLine block; + + public BlankLineParser(String raw) { + block = new BlankLine(raw); + } + + @Override + public Block getBlock() { + return block; + } + + @Override + public BlockContinue tryContinue(ParserState parserState) { + // Blank lines do not become active parsers, so they cannot be continued + return null; + } + +} \ No newline at end of file diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java index 493e4299c..47708e8f5 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java @@ -63,4 +63,49 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return usedDelimiters; } + + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + // "multiple of 3" rule for internal delimiter runs + if ((openingRun.canClose() || closingRun.canOpen()) && + closingRun.originalLength() % 3 != 0 && + (openingRun.originalLength() + closingRun.originalLength()) % 3 == 0) { + return 0; + } + + int usedDelimiters; + Node emphasis; + + // Pre-content whitespace in emphasis is primarily used during thematic breaks + String preContentWhitespace = prefix; + + if(!openingRun.getOpener().whitespacePreContent().isEmpty() && + openingRun.getOpener().whitespacePreContent().trim().equals("")) { + preContentWhitespace = openingRun.getOpener().whitespacePreContent(); + } + + // calculate actual number of delimiters used from this closer + if (openingRun.length() >= 2 && closingRun.length() >= 2) { + usedDelimiters = 2; + emphasis = new StrongEmphasis(String.valueOf(delimiterChar) + delimiterChar, preContentWhitespace); + } else { + usedDelimiters = 1; + emphasis = new Emphasis(String.valueOf(delimiterChar), preContentWhitespace); + } + + SourceSpans sourceSpans = SourceSpans.empty(); + sourceSpans.addAllFrom(openingRun.getOpeners(usedDelimiters)); + + Text opener = openingRun.getOpener(); + for (Node node : Nodes.between(opener, closingRun.getCloser())) { + emphasis.appendChild(node); + sourceSpans.addAll(node.getSourceSpans()); + } + + sourceSpans.addAllFrom(closingRun.getClosers(usedDelimiters)); + + emphasis.setSourceSpans(sourceSpans.getSourceSpans()); + opener.insertAfter(emphasis); + + return usedDelimiters; + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java index c29b8694f..b47b87546 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java @@ -45,9 +45,15 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Inline entities do not have significant prefix values + return tryParse(inlineParserState); + } private ParsedInline entity(Scanner scanner, Position start) { String text = scanner.getSource(start, scanner.position()).getContent(); - return ParsedInline.of(new Text(Html5Entities.entityToString(text)), scanner.position()); + return ParsedInline.of(new Text(Html5Entities.entityToString(text), text, "", ""), scanner.position()); } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java index 605901c22..14a773017 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java @@ -65,6 +65,12 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Inline HTML does not have significant prefix values + return tryParse(inlineParserState); + } private static ParsedInline htmlInline(Position start, Scanner scanner) { String text = scanner.getSource(start, scanner.position()).getContent(); diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java index 755ee3135..9fba0c949 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java @@ -3,4 +3,6 @@ public interface InlineContentParser { ParsedInline tryParse(InlineParserState inlineParserState); + + ParsedInline tryParse(InlineParserState inlineParserState, String prefix); } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java index 9de96a587..730de8682 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java @@ -1,12 +1,12 @@ package org.commonmark.internal.inline; +import java.util.List; + import org.commonmark.internal.util.CharMatcher; import org.commonmark.node.SourceSpan; import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import java.util.List; - public class Scanner { /** @@ -193,6 +193,30 @@ public int whitespace() { } } + /** + * Capture next series of whitespace + * @return Consecutive whitespace as String, or empty String if no whitespace found + */ + public String whitespaceAsString() { + StringBuilder sb = new StringBuilder(0); + while (true) { + char c = peek(); + switch (c) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + sb.append(c); + next(); + break; + default: + return sb.toString(); + } + } + } + public int find(char c) { int count = 0; while (true) { @@ -237,6 +261,10 @@ public void setPosition(Position position) { // For cases where the caller appends the result to a StringBuilder, we could offer another method to avoid some // unnecessary copying. public SourceLines getSource(Position begin, Position end) { + return getSource(begin, end, 0); + } + + public SourceLines getSource(Position begin, Position end, int offset) { if (begin.lineIndex == end.lineIndex) { // Shortcut for common case of text from a single line SourceLine line = lines.get(begin.lineIndex); @@ -244,7 +272,14 @@ public SourceLines getSource(Position begin, Position end) { SourceSpan newSourceSpan = null; SourceSpan sourceSpan = line.getSourceSpan(); if (sourceSpan != null) { - newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), sourceSpan.getColumnIndex() + begin.index, newContent.length()); + int newColumnIndex = sourceSpan.getColumnIndex() + begin.index; + + // Make sure SourceSpan is still tracking literal line, not its prefix + if(offset > 0) { + newColumnIndex -= offset; + } + + newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), newColumnIndex, newContent.length()); } return SourceLines.of(SourceLine.of(newContent, newSourceSpan)); } else { @@ -263,6 +298,18 @@ public SourceLines getSource(Position begin, Position end) { return sourceLines; } } + + // Many lines are passed around as raw strings, with an index value indicating + // where the literal string begins. Calling this method returns only the literal + // portion of the line, or an empty string if that is not possible. + public String alignToLiteral() { + if(index >= line.getLiteralIndex() || line.getLiteralIndex() == 0) { + return ""; + }else { + index = line.getLiteralIndex(); + return line.substring(0, index).getContent().toString(); + } + } private void setLine(SourceLine line) { this.line = line; diff --git a/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java b/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java index e02ecea7c..3bebf037d 100644 --- a/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java +++ b/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java @@ -5,11 +5,13 @@ public class OrderedListHolder extends ListHolder { private final char delimiter; private int counter; + private String rawNumber; public OrderedListHolder(ListHolder parent, OrderedList list) { super(parent); delimiter = list.getDelimiter(); counter = list.getStartNumber(); + rawNumber = list.getRawNumber(); } public char getDelimiter() { @@ -19,6 +21,10 @@ public char getDelimiter() { public int getCounter() { return counter; } + + public String getRawNumber() { + return rawNumber; + } public void increaseCounter() { counter++; diff --git a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java index 3ca34c5f0..c4513855d 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java @@ -70,7 +70,9 @@ public static boolean scanLinkTitle(Scanner scanner) { } char endDelimiter; - switch (scanner.peek()) { + char c = scanner.peek(); + + switch (c) { case '"': endDelimiter = '"'; break; @@ -83,6 +85,7 @@ public static boolean scanLinkTitle(Scanner scanner) { default: return false; } + scanner.next(); if (!scanLinkTitleContent(scanner, endDelimiter)) { @@ -158,6 +161,11 @@ private static boolean scanLinkDestinationWithBalancedParens(Scanner scanner) { } empty = false; } + + if(parens > 0) { + return false; + } + return true; } } diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java index 8b02e99b1..6d959c233 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java @@ -226,6 +226,56 @@ public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int last return lastIndex - 1; } + public static String collectWhitespace(CharSequence s, int startIndex, int lastIndex) { + StringBuilder whitespace = new StringBuilder(lastIndex - startIndex); + + for (int i = startIndex; i < lastIndex; i++) { + switch (s.charAt(i)) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + whitespace.append(s.charAt(i)); + break; + default: + return whitespace.toString(); + } + } + return whitespace.toString(); + } + + public static String collectWhitespaceBackwards(CharSequence s, int startIndex, int lastIndex) { + StringBuilder whitespace = new StringBuilder(startIndex - lastIndex); + + for (int i = startIndex; i >= lastIndex; i--) { + switch (s.charAt(i)) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + whitespace.append(s.charAt(i)); + break; + default: + return whitespace.toString(); + } + } + return whitespace.toString(); + } + + public static String generateSpaces(int numberOfSpaces) { + StringBuilder spaces = new StringBuilder(numberOfSpaces); + + for(int i = 0; i < numberOfSpaces; i++) { + spaces.append(" "); + } + + return spaces.toString(); + } + private static int findNonSpace(CharSequence s, int startIndex) { int length = s.length(); for (int i = startIndex; i < length; i++) { diff --git a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java index 7edd635d7..2d12e3a7c 100644 --- a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java +++ b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java @@ -8,6 +8,11 @@ */ public abstract class AbstractVisitor implements Visitor { + @Override + public void visit(BlankLine blankLine) { + visitChildren(blankLine); + } + @Override public void visit(BlockQuote blockQuote) { visitChildren(blockQuote); diff --git a/commonmark/src/main/java/org/commonmark/node/BlankLine.java b/commonmark/src/main/java/org/commonmark/node/BlankLine.java new file mode 100644 index 000000000..c60cc814d --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/node/BlankLine.java @@ -0,0 +1,23 @@ +package org.commonmark.node; + +public class BlankLine extends Block { + + private String raw; + + public BlankLine(String rawContent) { + raw = rawContent; + } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + @Override + public void accept(Visitor visitor) { + visitor.visit(this); + } +} diff --git a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java index 160f25ae2..707ca3ec3 100644 --- a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java +++ b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java @@ -1,9 +1,28 @@ package org.commonmark.node; public class BlockQuote extends Block { + // Whitespace for roundtrip rendering + private String whitespacePreMarker = ""; + private String whitespacePostMarker = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public String whitespacePreMarker() { + return whitespacePreMarker; + } + + public String whitespacePostMarker() { + return whitespacePostMarker; + } + + public void setPreMarkerWhitespace(String whitespace) { + whitespacePreMarker = whitespace; + } + + public void setPostMarkerWhitespace(String whitespace) { + whitespacePostMarker = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/BulletList.java b/commonmark/src/main/java/org/commonmark/node/BulletList.java index 127862312..b0db34bba 100644 --- a/commonmark/src/main/java/org/commonmark/node/BulletList.java +++ b/commonmark/src/main/java/org/commonmark/node/BulletList.java @@ -1,7 +1,6 @@ package org.commonmark.node; public class BulletList extends ListBlock { - private char bulletMarker; @Override @@ -16,5 +15,4 @@ public char getBulletMarker() { public void setBulletMarker(char bulletMarker) { this.bulletMarker = bulletMarker; } - } diff --git a/commonmark/src/main/java/org/commonmark/node/Code.java b/commonmark/src/main/java/org/commonmark/node/Code.java index 0b47ecb71..268667978 100644 --- a/commonmark/src/main/java/org/commonmark/node/Code.java +++ b/commonmark/src/main/java/org/commonmark/node/Code.java @@ -3,6 +3,9 @@ public class Code extends Node { private String literal; + private String raw; + private boolean strippedSpaces; + private int numBackticks; public Code() { } @@ -23,4 +26,28 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public boolean hasStrippedSpaces() { + return strippedSpaces; + } + + public void setStrippedSpaces(boolean strippedSpaces) { + this.strippedSpaces = strippedSpaces; + } + + public int getNumBackticks() { + return numBackticks; + } + + public void setNumBackticks(int numBackticks) { + this.numBackticks = numBackticks; + } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Document.java b/commonmark/src/main/java/org/commonmark/node/Document.java index 5b7e74189..cfa18c488 100644 --- a/commonmark/src/main/java/org/commonmark/node/Document.java +++ b/commonmark/src/main/java/org/commonmark/node/Document.java @@ -1,9 +1,18 @@ package org.commonmark.node; public class Document extends Block { + private String whitespaceEndOfDocument = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public String whitespaceEndOfDocument() { + return whitespaceEndOfDocument; + } + + public void setEndOfDocumentWhitespace(String whitespace) { + whitespaceEndOfDocument = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Emphasis.java b/commonmark/src/main/java/org/commonmark/node/Emphasis.java index 9877e7b63..54acaf7c5 100644 --- a/commonmark/src/main/java/org/commonmark/node/Emphasis.java +++ b/commonmark/src/main/java/org/commonmark/node/Emphasis.java @@ -3,6 +3,7 @@ public class Emphasis extends Node implements Delimited { private String delimiter; + private String preBlockWhitespace = ""; public Emphasis() { } @@ -10,6 +11,15 @@ public Emphasis() { public Emphasis(String delimiter) { this.delimiter = delimiter; } + + public Emphasis(String delimiter, String preBlockWhitespace) { + this(delimiter); + this.preBlockWhitespace = preBlockWhitespace; + } + + public String whitespacePreBlock() { + return preBlockWhitespace; + } public void setDelimiter(String delimiter) { this.delimiter = delimiter; diff --git a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java index 7e2612331..b56d8952c 100644 --- a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java @@ -3,11 +3,18 @@ public class FencedCodeBlock extends Block { private char fenceChar; - private int fenceLength; - private int fenceIndent; + private int startFenceLength; + private int endFenceLength; private String info; private String literal; + private String raw; + + // Whitespace tracked for roundtrip rendering + private String whitespacePreStartFence = ""; + private String whitespacePreContent = ""; + private String whitespacePreEndFence = ""; + private String whitespacePostBlock = ""; @Override public void accept(Visitor visitor) { @@ -22,24 +29,33 @@ public void setFenceChar(char fenceChar) { this.fenceChar = fenceChar; } - public int getFenceLength() { - return fenceLength; + public int getStartFenceLength() { + return startFenceLength; } - public void setFenceLength(int fenceLength) { - this.fenceLength = fenceLength; + public void setStartFenceLength(int startFenceLength) { + this.startFenceLength = startFenceLength; } - public int getFenceIndent() { - return fenceIndent; + public int getEndFenceLength() { + return endFenceLength; } - public void setFenceIndent(int fenceIndent) { - this.fenceIndent = fenceIndent; + public void setEndFenceLength(int endFenceLength) { + this.endFenceLength = endFenceLength; + } + + public int getStartFenceIndent() { + return whitespacePreStartFence.length(); + } + + public int getEndFenceIndent() { + return whitespacePreEndFence.length(); } /** * @see CommonMark spec + * @return Info string or null */ public String getInfo() { return info; @@ -56,4 +72,44 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + public String whitespacePreStartFence() { + return whitespacePreStartFence; + } + + public String whitespacePreContent() { + return whitespacePreContent; + } + + public String whitespacePreEndFence() { + return whitespacePreEndFence; + } + + public String whitespacePostBlock() { + return whitespacePostBlock; + } + + public void setPreStartFenceWhitespace(String whitespace) { + whitespacePreStartFence = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } + + public void setPreEndFenceWhitespace(String whitespace) { + whitespacePreEndFence = whitespace; + } + + public void setPostBlockWhitespace(String whitespace) { + whitespacePostBlock = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java b/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java index 0640fc3c4..72e9ad8ad 100644 --- a/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java +++ b/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java @@ -1,9 +1,23 @@ package org.commonmark.node; public class HardLineBreak extends Node { - + boolean hasBackslash = false; + + // Preserve original default constructor by explicitly defining one + public HardLineBreak() { + super(); + } + + public HardLineBreak(boolean hasBackslash) { + this.hasBackslash = hasBackslash; + } + @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public boolean hasBackslash() { + return hasBackslash; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Heading.java b/commonmark/src/main/java/org/commonmark/node/Heading.java index 41f3b2504..a5f37b64d 100644 --- a/commonmark/src/main/java/org/commonmark/node/Heading.java +++ b/commonmark/src/main/java/org/commonmark/node/Heading.java @@ -1,8 +1,19 @@ package org.commonmark.node; public class Heading extends Block { - private int level; + private char symbolType; + private int numEndingSymbol; + + // Whitespace for roundtrip rendering + // AST: Pre-marker whitespace (ATX) or before setext text (setext) + private String whitespacePreBlock = ""; + // AST: Pre-text (ATX) or after setext text ends (setext) + private String whitespacePreContent = ""; + // AST: Post-text (ATX) or before setext delimiting line (setext) + private String whitespacePostContent = ""; + // AST: Post-block (ATX) or after setext delimiting line (setext) + private String whitespacePostBlock = ""; @Override public void accept(Visitor visitor) { @@ -16,4 +27,52 @@ public int getLevel() { public void setLevel(int level) { this.level = level; } + + public char getSymbolType() { + return symbolType; + } + + public void setSymbolType(char symbolType) { + this.symbolType = symbolType; + } + + public int getNumEndingSymbol() { + return numEndingSymbol; + } + + public void setNumEndingSymbol(int numEndingSymbol) { + this.numEndingSymbol = numEndingSymbol; + } + + public String whitespacePreBlock() { + return whitespacePreBlock; + } + + public String whitespacePreContent() { + return whitespacePreContent; + } + + public String whitespacePostContent() { + return whitespacePostContent; + } + + public String whitespacePostBlock() { + return whitespacePostBlock; + } + + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } + + public void setPostContentWhitespace(String whitespace) { + whitespacePostContent = whitespace; + } + + public void setPostBlockWhitespace(String whitespace) { + whitespacePostBlock = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java index ad46c56ce..6677ecacf 100644 --- a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java @@ -6,8 +6,8 @@ * @see CommonMark Spec */ public class HtmlBlock extends Block { - private String literal; + private String raw; @Override public void accept(Visitor visitor) { @@ -21,4 +21,12 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HtmlInline.java b/commonmark/src/main/java/org/commonmark/node/HtmlInline.java index 291fcde3c..86caf9ce1 100644 --- a/commonmark/src/main/java/org/commonmark/node/HtmlInline.java +++ b/commonmark/src/main/java/org/commonmark/node/HtmlInline.java @@ -8,6 +8,7 @@ public class HtmlInline extends Node { private String literal; + private String raw; @Override public void accept(Visitor visitor) { @@ -21,4 +22,12 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Image.java b/commonmark/src/main/java/org/commonmark/node/Image.java index 63481773a..fdc1fcfad 100644 --- a/commonmark/src/main/java/org/commonmark/node/Image.java +++ b/commonmark/src/main/java/org/commonmark/node/Image.java @@ -1,16 +1,50 @@ package org.commonmark.node; -public class Image extends Node { +public class Image extends Node implements LinkFormat { private String destination; + private String rawDestination; private String title; + private String rawTitle; + private String label; + private String rawLabel; + private LinkType linkType; + private char titleSymbol; + private String whitespacePreDestination; + private String whitespacePreTitle; + private String whitespacePostContent; public Image() { } public Image(String destination, String title) { this.destination = destination; + this.rawDestination = ""; this.title = title; + this.rawTitle = ""; + this.label = null; + this.rawLabel = null; + linkType = LinkType.NULL; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; + } + + public Image(String destination, String rawDestination, String title, + String rawTitle, String label, String rawLabel, LinkType linkType, + char titleSymbol, String whitespacePreDestination, + String whitespacePreTitle, String whitespacePostContent) { + this.destination = destination; + this.rawDestination = rawDestination; + this.title = title; + this.rawTitle = rawTitle; + this.label = label; + this.rawLabel = rawLabel; + this.linkType = linkType; + this.titleSymbol = titleSymbol; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostContent = whitespacePostContent; } @Override @@ -18,21 +52,115 @@ public void accept(Visitor visitor) { visitor.visit(this); } + @Override public String getDestination() { return destination; } + @Override public void setDestination(String destination) { this.destination = destination; } + + @Override + public String getRawDestination() { + return rawDestination; + } + @Override + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + + @Override public String getTitle() { return title; } + @Override public void setTitle(String title) { this.title = title; } + + @Override + public String getRawTitle() { + return rawTitle; + } + + @Override + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + @Override + public LinkType getLinkType() { + return linkType; + } + + @Override + public void setLinkType(LinkType linkType) { + this.linkType = linkType; + } + + @Override + public char getTitleSymbol() { + return titleSymbol; + } + + @Override + public void setTitleSymbol(char titleSymbol) { + this.titleSymbol = titleSymbol; + } + + @Override + public String getWhitespacePreDestination() { + return whitespacePreDestination; + } + + @Override + public void setWhitespacePreDestination(String whitespacePreDestination) { + this.whitespacePreDestination = whitespacePreDestination; + } + + @Override + public String getWhitespacePreTitle() { + return whitespacePreTitle; + } + + @Override + public void setWhitespacePreTitle(String whitespacePreTitle) { + this.whitespacePreTitle = whitespacePreTitle; + } + + @Override + public String getWhitespacePostContent() { + return whitespacePostContent; + } + + @Override + public void setWhitespacePostContent(String whitespacePostContent) { + this.whitespacePostContent = whitespacePostContent; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public void setLabel(String label) { + this.label = label; + } + + @Override + public String getRawLabel() { + return rawLabel; + } + + @Override + public void setRawLabel(String rawLabel) { + this.rawLabel = rawLabel; + } @Override protected String toStringAttributes() { diff --git a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java index ccafca943..878e0af32 100644 --- a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java @@ -1,8 +1,16 @@ package org.commonmark.node; public class IndentedCodeBlock extends Block { - + // The "literal" string is optimized for HTML formatting, but omits some details + // which are important for roundtrip rendering private String literal; + + // The "raw" string is, as much as possible, the entire raw content as it was first + // entered into the parser. This allows roundtrip parsing for indented code blocks. + private String raw; + + // Actual indent whitespace of a raw string for roundtrip rendering + private String indentWhitespace = ""; @Override public void accept(Visitor visitor) { @@ -16,4 +24,20 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + public String getIndentWhitespace() { + return indentWhitespace; + } + + public void setIndentWhitespace(String whitespace) { + indentWhitespace = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Link.java b/commonmark/src/main/java/org/commonmark/node/Link.java index b2ed8c2a1..5a7864542 100644 --- a/commonmark/src/main/java/org/commonmark/node/Link.java +++ b/commonmark/src/main/java/org/commonmark/node/Link.java @@ -20,17 +20,51 @@ * * @see CommonMark Spec for links */ -public class Link extends Node { +public class Link extends Node implements LinkFormat { private String destination; + private String rawDestination; private String title; + private String rawTitle; + private String label; + private String rawLabel; + private LinkType linkType; + private char titleSymbol; + private String whitespacePreDestination; + private String whitespacePreTitle; + private String whitespacePostContent; public Link() { } public Link(String destination, String title) { this.destination = destination; + this.rawDestination = ""; this.title = title; + this.rawTitle = ""; + this.label = ""; + this.rawLabel = ""; + linkType = LinkType.NULL; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; + } + + public Link(String destination, String rawDestination, String title, + String rawTitle, String label, String rawLabel, + LinkType linkType, char titleSymbol, String whitespacePreDestination, + String whitespacePreTitle, String whitespacePostContent) { + this.destination = destination; + this.rawDestination = rawDestination; + this.title = title; + this.rawTitle = rawTitle; + this.label = label; + this.rawLabel = rawLabel; + this.linkType = linkType; + this.titleSymbol = titleSymbol; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostContent = whitespacePostContent; } @Override @@ -38,22 +72,116 @@ public void accept(Visitor visitor) { visitor.visit(this); } + @Override public String getDestination() { return destination; } + @Override public void setDestination(String destination) { this.destination = destination; } + + @Override + public String getRawDestination() { + return rawDestination; + } + @Override + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + + @Override public String getTitle() { return title; } + @Override public void setTitle(String title) { this.title = title; } + + @Override + public String getRawTitle() { + return rawTitle; + } + + @Override + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + @Override + public LinkType getLinkType() { + return linkType; + } + + @Override + public void setLinkType(LinkType linkType) { + this.linkType = linkType; + } + @Override + public char getTitleSymbol() { + return titleSymbol; + } + + @Override + public void setTitleSymbol(char titleSymbol) { + this.titleSymbol = titleSymbol; + } + + @Override + public String getWhitespacePreDestination() { + return whitespacePreDestination; + } + + @Override + public void setWhitespacePreDestination(String whitespacePreDestination) { + this.whitespacePreDestination = whitespacePreDestination; + } + + @Override + public String getWhitespacePreTitle() { + return whitespacePreTitle; + } + + @Override + public void setWhitespacePreTitle(String whitespacePreTitle) { + this.whitespacePreTitle = whitespacePreTitle; + } + + @Override + public String getWhitespacePostContent() { + return whitespacePostContent; + } + + @Override + public void setWhitespacePostContent(String whitespacePostContent) { + this.whitespacePostContent = whitespacePostContent; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public void setLabel(String label) { + this.label = label; + } + + @Override + public String getRawLabel() { + return rawLabel; + } + + @Override + public void setRawLabel(String rawLabel) { + this.rawLabel = rawLabel; + } + @Override protected String toStringAttributes() { return "destination=" + destination + ", title=" + title; diff --git a/commonmark/src/main/java/org/commonmark/node/LinkFormat.java b/commonmark/src/main/java/org/commonmark/node/LinkFormat.java new file mode 100644 index 000000000..267bcfa3d --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/node/LinkFormat.java @@ -0,0 +1,30 @@ +package org.commonmark.node; + +public interface LinkFormat { + public String getDestination(); + public void setDestination(String destination); + public String getRawDestination(); + public void setRawDestination(String destination); + public String getLabel(); + public void setLabel(String label); + public String getRawLabel(); + public void setRawLabel(String label); + public String getTitle(); + public void setTitle(String title); + public String getRawTitle(); + public void setRawTitle(String title); + public char getTitleSymbol(); + public void setTitleSymbol(char symbol); + public LinkType getLinkType(); + public void setLinkType(LinkType linkType); + public String getWhitespacePreDestination(); + public void setWhitespacePreDestination(String whitespace); + public String getWhitespacePreTitle(); + public void setWhitespacePreTitle(String whitespace); + public String getWhitespacePostContent(); + public void setWhitespacePostContent(String whitespace); + + public enum LinkType { + INLINE, REFERENCE, AUTOLINK, NULL; + } +} diff --git a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java index 3f8bfd0f0..a47d007b3 100644 --- a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java +++ b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java @@ -15,15 +15,29 @@ public class LinkReferenceDefinition extends Node { private String label; private String destination; + private String rawDestination; private String title; + private String rawTitle; + private char delimiterChar; + private String whitespacePreLabel = ""; + private String whitespacePreDestination = ""; + private String whitespacePreTitle = ""; + private String whitespacePostTitle = ""; public LinkReferenceDefinition() { } - public LinkReferenceDefinition(String label, String destination, String title) { + public LinkReferenceDefinition(String label, String destination, String rawDestination, String title, String rawTitle, char delimiterChar, String whitespacePreLabel, String whitespacePreDestination, String whitespacePreTitle, String whitespacePostTitle) { this.label = label; this.destination = destination; + this.rawDestination = rawDestination; this.title = title; + this.rawTitle = rawTitle; + this.delimiterChar = delimiterChar; + this.whitespacePreLabel = whitespacePreLabel; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostTitle = whitespacePostTitle; } public String getLabel() { @@ -42,6 +56,14 @@ public void setDestination(String destination) { this.destination = destination; } + public String getRawDestination() { + return rawDestination; + } + + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + public String getTitle() { return title; } @@ -49,6 +71,34 @@ public String getTitle() { public void setTitle(String title) { this.title = title; } + + public String getRawTitle() { + return rawTitle; + } + + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + public char getDelimiterChar() { + return delimiterChar; + } + + public String whitespacePreLabel() { + return whitespacePreLabel; + } + + public String whitespacePreDestination() { + return whitespacePreDestination; + } + + public String whitespacePreTitle() { + return whitespacePreTitle; + } + + public String whitespacePostTitle() { + return whitespacePostTitle; + } @Override public void accept(Visitor visitor) { diff --git a/commonmark/src/main/java/org/commonmark/node/ListBlock.java b/commonmark/src/main/java/org/commonmark/node/ListBlock.java index 69482f66e..3f8f129dd 100644 --- a/commonmark/src/main/java/org/commonmark/node/ListBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/ListBlock.java @@ -3,6 +3,10 @@ public abstract class ListBlock extends Block { private boolean tight; + + // Whitespace for roundtrip parsing of the first line in a list block + private String whitespacePreBlock = ""; + private String whitespacePreContent = ""; /** * @return whether this list is tight or loose @@ -16,4 +20,19 @@ public void setTight(boolean tight) { this.tight = tight; } + public String whitespacePreBlock() { + return whitespacePreBlock; + } + + public String whitespacePreContent() { + return whitespacePreContent; + } + + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/ListItem.java b/commonmark/src/main/java/org/commonmark/node/ListItem.java index aa526be01..0374bdd81 100644 --- a/commonmark/src/main/java/org/commonmark/node/ListItem.java +++ b/commonmark/src/main/java/org/commonmark/node/ListItem.java @@ -1,9 +1,37 @@ package org.commonmark.node; public class ListItem extends Block { + private String rawNumber = ""; + + private String whitespacePreMarker = ""; + private String whitespacePostMarker = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public String whitespacePreMarker() { + return whitespacePreMarker; + } + + public String whitespacePostMarker() { + return whitespacePostMarker; + } + + public void setPreMarkerWhitespace(String whitespace) { + whitespacePreMarker = whitespace; + } + + public void setPostMarkerWhitespace(String whitespace) { + whitespacePostMarker = whitespace; + } + + public String getRawNumber() { + return rawNumber; + } + + public void setRawNumber(String rawNumber) { + this.rawNumber = rawNumber; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Node.java b/commonmark/src/main/java/org/commonmark/node/Node.java index 5a2f036e4..d3dff717a 100644 --- a/commonmark/src/main/java/org/commonmark/node/Node.java +++ b/commonmark/src/main/java/org/commonmark/node/Node.java @@ -149,7 +149,7 @@ public void addSourceSpan(SourceSpan sourceSpan) { } this.sourceSpans.add(sourceSpan); } - + @Override public String toString() { return getClass().getSimpleName() + "{" + toStringAttributes() + "}"; diff --git a/commonmark/src/main/java/org/commonmark/node/Nodes.java b/commonmark/src/main/java/org/commonmark/node/Nodes.java index 22d5932af..da99d63ad 100644 --- a/commonmark/src/main/java/org/commonmark/node/Nodes.java +++ b/commonmark/src/main/java/org/commonmark/node/Nodes.java @@ -14,6 +14,9 @@ private Nodes() { /** * The nodes between (not including) start and end. + * @param start Start node + * @param end End node + * @return Iterable list of 
Node
s */ public static Iterable between(Node start, Node end) { return new NodeIterable(start.getNext(), end); diff --git a/commonmark/src/main/java/org/commonmark/node/OrderedList.java b/commonmark/src/main/java/org/commonmark/node/OrderedList.java index 1f988234c..d9a1fb51d 100644 --- a/commonmark/src/main/java/org/commonmark/node/OrderedList.java +++ b/commonmark/src/main/java/org/commonmark/node/OrderedList.java @@ -1,8 +1,8 @@ package org.commonmark.node; public class OrderedList extends ListBlock { - private int startNumber; + private String rawNumber; private char delimiter; @Override @@ -17,6 +17,14 @@ public int getStartNumber() { public void setStartNumber(int startNumber) { this.startNumber = startNumber; } + + public String getRawNumber() { + return rawNumber; + } + + public void setRawNumber(String rawNumber) { + this.rawNumber = rawNumber; + } public char getDelimiter() { return delimiter; @@ -25,5 +33,4 @@ public char getDelimiter() { public void setDelimiter(char delimiter) { this.delimiter = delimiter; } - } diff --git a/commonmark/src/main/java/org/commonmark/node/Paragraph.java b/commonmark/src/main/java/org/commonmark/node/Paragraph.java index 176eaaa76..773b14c45 100644 --- a/commonmark/src/main/java/org/commonmark/node/Paragraph.java +++ b/commonmark/src/main/java/org/commonmark/node/Paragraph.java @@ -4,9 +4,19 @@ * A paragraph block, contains inline nodes such as {@link Text} */ public class Paragraph extends Block { + // Whitespace for roundtrip rendering + private String whitespacePreBlock = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public String whitespacePreBlock() { + return whitespacePreBlock; + } + + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java b/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java index dbff571cd..939944dd8 100644 --- a/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java +++ b/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java @@ -4,12 +4,24 @@ public class StrongEmphasis extends Node implements Delimited { private String delimiter; + // Specifically used during roundtrip rendering of thematic breaks + private String preBlockWhitespace; + public StrongEmphasis() { } public StrongEmphasis(String delimiter) { this.delimiter = delimiter; } + + public StrongEmphasis(String delimiter, String preBlockWhitespace) { + this(delimiter); + this.preBlockWhitespace = preBlockWhitespace; + } + + public String whitespacePreBlock() { + return preBlockWhitespace; + } public void setDelimiter(String delimiter) { this.delimiter = delimiter; diff --git a/commonmark/src/main/java/org/commonmark/node/Text.java b/commonmark/src/main/java/org/commonmark/node/Text.java index f16fc907b..8edb88093 100644 --- a/commonmark/src/main/java/org/commonmark/node/Text.java +++ b/commonmark/src/main/java/org/commonmark/node/Text.java @@ -1,14 +1,29 @@ package org.commonmark.node; +import java.util.Objects; + public class Text extends Node { private String literal; + private String raw = ""; + private String preContentWhitespace; + private String postContentWhitespace; public Text() { } public Text(String literal) { this.literal = literal; + this.raw = literal; + this.preContentWhitespace = ""; + this.postContentWhitespace = ""; + } + + public Text(String literal, String raw, String preContentWhitespace, String postContentWhitespace) { + this.literal = literal; + this.raw = raw; + this.preContentWhitespace = preContentWhitespace; + this.postContentWhitespace = postContentWhitespace; } @Override @@ -23,9 +38,56 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + public String whitespacePreContent() { + if(preContentWhitespace != null) { + return preContentWhitespace; + }else { + return ""; + } + } + + public String whitespacePostContent() { + if(postContentWhitespace != null) { + return postContentWhitespace; + }else { + return ""; + } + } + + public void setWhitespace(String pre, String post) { + preContentWhitespace = pre; + postContentWhitespace = post; + } @Override protected String toStringAttributes() { return "literal=" + literal; } + + @Override + public boolean equals(Object obj) { + if(obj instanceof Text) { + if(((Text)obj).getLiteral().equals(literal) && ((Text)obj).getRaw().equals(raw) && + ((Text)obj).whitespacePreContent().equals(preContentWhitespace) && + ((Text)obj).whitespacePostContent().equals(postContentWhitespace)) { + return true; + } + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hash(literal, raw, preContentWhitespace, postContentWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java index f81abaa31..bcd66072d 100644 --- a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java +++ b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java @@ -1,9 +1,18 @@ package org.commonmark.node; public class ThematicBreak extends Block { + private CharSequence content; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public CharSequence getContent() { + return content; + } + + public void setContent(CharSequence content) { + this.content = content; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Visitor.java b/commonmark/src/main/java/org/commonmark/node/Visitor.java index a155296f0..f3b6def65 100644 --- a/commonmark/src/main/java/org/commonmark/node/Visitor.java +++ b/commonmark/src/main/java/org/commonmark/node/Visitor.java @@ -7,6 +7,8 @@ */ public interface Visitor { + void visit(BlankLine blankLine); + void visit(BlockQuote blockQuote); void visit(BulletList bulletList); diff --git a/commonmark/src/main/java/org/commonmark/parser/SourceLine.java b/commonmark/src/main/java/org/commonmark/parser/SourceLine.java index 63caceb9e..bdbb3dbda 100644 --- a/commonmark/src/main/java/org/commonmark/parser/SourceLine.java +++ b/commonmark/src/main/java/org/commonmark/parser/SourceLine.java @@ -1,5 +1,7 @@ package org.commonmark.parser; +import java.util.Objects; + import org.commonmark.node.SourceSpan; /** @@ -11,10 +13,16 @@ public class SourceLine { private final CharSequence content; private final SourceSpan sourceSpan; + private final int literalIndex; public static SourceLine of(CharSequence content, SourceSpan sourceSpan) { return new SourceLine(content, sourceSpan); } + + // The literal index is the beginning of a literal line, as found within a raw line + public static SourceLine of(CharSequence content, SourceSpan sourceSpan, int literalIndex) { + return new SourceLine(content, sourceSpan, literalIndex); + } private SourceLine(CharSequence content, SourceSpan sourceSpan) { if (content == null) { @@ -22,6 +30,16 @@ private SourceLine(CharSequence content, SourceSpan sourceSpan) { } this.content = content; this.sourceSpan = sourceSpan; + this.literalIndex = 0; + } + + private SourceLine(CharSequence content, SourceSpan sourceSpan, int literalIndex) { + if (content == null) { + throw new NullPointerException("content must not be null"); + } + this.content = content; + this.sourceSpan = sourceSpan; + this.literalIndex = literalIndex; } public CharSequence getContent() { @@ -31,6 +49,10 @@ public CharSequence getContent() { public SourceSpan getSourceSpan() { return sourceSpan; } + + public int getLiteralIndex() { + return literalIndex; + } public SourceLine substring(int beginIndex, int endIndex) { CharSequence newContent = content.subSequence(beginIndex, endIndex); @@ -44,4 +66,41 @@ public SourceLine substring(int beginIndex, int endIndex) { } return SourceLine.of(newContent, newSourceSpan); } + + // Many lines are passed around as raw strings, with an index value indicating + // where the literal string begins. Calling this method returns a SourceLine containing + // only the literal portion of the line (if it is a subset of the line) or the line + // in full (if they are the same). + public SourceLine getLiteralLine() { + if(literalIndex == 0) { + return SourceLine.of(content, sourceSpan); + }else { + SourceSpan newSourceSpan = null; + if(sourceSpan != null) { + int columnIndex = sourceSpan.getColumnIndex() - literalIndex; + int length = content.length() - literalIndex; + if(length != 0) { + newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), columnIndex, length); + } + } + return SourceLine.of(content.subSequence(literalIndex, content.length()), newSourceSpan); + } + } + + @Override + public int hashCode() { + return Objects.hash(content, literalIndex); + } + + @Override + public boolean equals(Object obj) { + if(obj instanceof SourceLine) { + if(((SourceLine)obj).getContent().toString().equals(content) && + ((SourceLine)obj).getLiteralIndex() == literalIndex) { + return true; + } + } + + return false; + } } diff --git a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java index 0b4290341..76dc25355 100644 --- a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java +++ b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java @@ -1,9 +1,10 @@ package org.commonmark.parser; -import org.commonmark.node.SourceSpan; - import java.util.ArrayList; import java.util.List; +import java.util.Objects; + +import org.commonmark.node.SourceSpan; /** * A set of lines ({@link SourceLine}) from the input source. @@ -63,4 +64,33 @@ public List getSourceSpans() { } return sourceSpans; } + + /** + * Return original 
List<SourceLine>
, but with first line removed. + * If the list is already empty, return the list unchanged + * @return List of source lines, or a blank list + */ + public List removeFirstLine() { + if(!isEmpty()) { + lines.remove(0); + } + + return lines; + } + + @Override + public int hashCode() { + return Objects.hash(lines); + } + + @Override + public boolean equals(Object obj) { + if(obj instanceof SourceLines) { + if(((SourceLines)obj).getLines().equals(lines)) { + return true; + } + } + + return false; + } } diff --git a/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java b/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java index aa956a48a..7c8358f7a 100644 --- a/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java @@ -13,18 +13,18 @@ public interface BlockParser { /** - * Return true if the block that is parsed is a container (contains other blocks), or false if it's a leaf. + * @return true if the block that is parsed is a container (contains other blocks), or false if it's a leaf. */ boolean isContainer(); /** - * Return true if the block can have lazy continuation lines. *

* Lazy continuation lines are lines that were rejected by this {@link #tryContinue(ParserState)} but didn't match * any other block parsers either. *

* If true is returned here, those lines will get added via {@link #addLine(SourceLine)}. For false, the block is * closed instead. + * @return true if the block can have lazy continuation lines. */ boolean canHaveLazyContinuationLines(); @@ -42,6 +42,7 @@ public interface BlockParser { * need to override this. * * @since 0.16.0 + * @param sourceSpan Source span to be added into the current block */ void addSourceSpan(SourceSpan sourceSpan); diff --git a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java index 897943d66..00b51890e 100644 --- a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java @@ -23,6 +23,7 @@ public interface DelimiterProcessor { /** * Minimum number of delimiter characters that are needed to activate this. Must be at least 1. + * @return the minimum number of delimiter characters needed to activate the processor. */ int getMinLength(); @@ -41,4 +42,19 @@ public interface DelimiterProcessor { */ int process(DelimiterRun openingRun, DelimiterRun closingRun); + /** + * Process the delimiter runs. + *

+ * The processor can examine the runs and the nodes and decide if it wants to process or not. If not, it should not + * change any nodes and return 0. If yes, it should do the processing (wrapping nodes, etc) and then return how many + * delimiters were used. + *

+ * Note that removal (unlinking) of the used delimiter {@link Text} nodes is done by the caller. + * + * @param openingRun the opening delimiter run + * @param closingRun the closing delimiter run + * @param prefix whitespace before the delimiter run, if any + * @return how many delimiters were used; must not be greater than length of either opener or closer + */ + int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix); } diff --git a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java index 578eac96b..dea13f4c9 100644 --- a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java +++ b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java @@ -44,6 +44,8 @@ public interface DelimiterRun { *

* For example, for a delimiter run {@code ***}, calling this with 1 would return the last {@code *}. * Calling it with 2 would return the second last {@code *} and the last {@code *}. + * @param length Number of openers to capture + * @return Requested number of opening characters */ Iterable getOpeners(int length); @@ -53,6 +55,8 @@ public interface DelimiterRun { *

* For example, for a delimiter run {@code ***}, calling this with 1 would return the first {@code *}. * Calling it with 2 would return the first {@code *} and the second {@code *}. + * @param length Number of closers to capture + * @return Requested number of closing characters */ Iterable getClosers(int length); } diff --git a/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java index 7d3552668..54756895f 100644 --- a/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java +++ b/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java @@ -1,10 +1,39 @@ package org.commonmark.renderer.html; -import org.commonmark.node.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.util.Escaping; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.BlankLine; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Document; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; import org.commonmark.renderer.NodeRenderer; -import java.util.*; - /** * The node renderer that renders all the core nodes (comes last in the order of node renderers). */ @@ -40,7 +69,8 @@ public Set> getNodeTypes() { Code.class, HtmlInline.class, SoftLineBreak.class, - HardLineBreak.class + HardLineBreak.class, + BlankLine.class )); } @@ -99,7 +129,11 @@ public void visit(BulletList bulletList) { public void visit(FencedCodeBlock fencedCodeBlock) { String literal = fencedCodeBlock.getLiteral(); Map attributes = new LinkedHashMap<>(); - String info = fencedCodeBlock.getInfo(); + + // The info string for FencedCodeBlock is not escaped or trimmed + // of excess whitespace, do that now + String info = Escaping.unescapeString(fencedCodeBlock.getInfo()).trim(); + if (info != null && !info.isEmpty()) { int space = info.indexOf(" "); String language; @@ -244,6 +278,13 @@ public void visit(HardLineBreak hardLineBreak) { html.line(); } + @Override + public void visit(BlankLine blankLine) { + // The BlankLine class is not part of the CommonMark standard, and is + // only for roundtrip rendering. Thus, it is specifically ignored + // by the HTML renderer. + } + @Override protected void visitChildren(Node parent) { Node node = parent.getFirstChild(); diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java new file mode 100644 index 000000000..bf0778464 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java @@ -0,0 +1,25 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.node.Node; + +public interface CommonMarkNodeRendererContext { + + /** + * @return true for stripping new lines and render text as "single line", + * false for keeping all line breaks. + */ + boolean stripNewlines(); + + /** + * @return the writer to use + */ + CommonMarkWriter getWriter(); + + /** + * Render the specified node and its children using the configured renderers. This should be used to render child + * nodes; be careful not to pass the node that is being rendered, that would result in an endless loop. + * + * @param node the node to render + */ + void render(Node node); +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java new file mode 100644 index 000000000..d24336334 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java @@ -0,0 +1,149 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.Extension; +import org.commonmark.internal.renderer.NodeRendererMap; +import org.commonmark.node.Node; +import org.commonmark.renderer.NodeRenderer; +import org.commonmark.renderer.Renderer; + +import java.util.ArrayList; +import java.util.List; + +public class CommonMarkRenderer implements Renderer { + + private final boolean stripNewlines; + + private final List nodeRendererFactories; + + private CommonMarkRenderer(Builder builder) { + this.stripNewlines = builder.stripNewlines; + + this.nodeRendererFactories = new ArrayList<>(builder.nodeRendererFactories.size() + 1); + this.nodeRendererFactories.addAll(builder.nodeRendererFactories); + // Add as last. This means clients can override the rendering of core nodes if they want. + this.nodeRendererFactories.add(new CommonmarkNodeRendererFactory() { + @Override + public NodeRenderer create(CommonMarkNodeRendererContext context) { + return new CoreCommonMarkNodeRenderer(context); + } + }); + } + + /** + * Create a new builder for configuring an {@link CommonMarkRenderer}. + * + * @return a builder + */ + public static Builder builder() { + return new Builder(); + } + + @Override + public void render(Node node, Appendable output) { + RendererContext context = new RendererContext(new CommonMarkWriter(output)); + context.render(node); + } + + @Override + public String render(Node node) { + StringBuilder sb = new StringBuilder(); + render(node, sb); + return sb.toString(); + } + + /** + * Builder for configuring an {@link CommonMarkRenderer}. See methods for default configuration. + */ + public static class Builder { + + private boolean stripNewlines = false; + private List nodeRendererFactories = new ArrayList<>(); + + /** + * @return the configured {@link CommonMarkRenderer} + */ + public CommonMarkRenderer build() { + return new CommonMarkRenderer(this); + } + + /** + * Set the value of flag for stripping new lines. + * + * @param stripNewlines true for stripping new lines and render text as "single line", + * false for keeping all line breaks + * @return {@code this} + */ + public Builder stripNewlines(boolean stripNewlines) { + this.stripNewlines = stripNewlines; + return this; + } + + /** + * Add a factory for instantiating a node renderer (done when rendering). This allows to override the rendering + * of node types or define rendering for custom node types. + *

+ * If multiple node renderers for the same node type are created, the one from the factory that was added first + * "wins". (This is how the rendering for core node types can be overridden; the default rendering comes last.) + * + * @param nodeRendererFactory the factory for creating a node renderer + * @return {@code this} + */ + public Builder nodeRendererFactory(CommonmarkNodeRendererFactory nodeRendererFactory) { + this.nodeRendererFactories.add(nodeRendererFactory); + return this; + } + + /** + * @param extensions extensions to use on this text content renderer + * @return {@code this} + */ + public Builder extensions(Iterable extensions) { + for (Extension extension : extensions) { + if (extension instanceof CommonMarkRenderer.CommonMarkRendererExtension) { + CommonMarkRenderer.CommonMarkRendererExtension textContentRendererExtension = + (CommonMarkRenderer.CommonMarkRendererExtension) extension; + textContentRendererExtension.extend(this); + } + } + return this; + } + } + + /** + * Extension for {@link CommonMarkRenderer}. + */ + public interface CommonMarkRendererExtension extends Extension { + void extend(CommonMarkRenderer.Builder rendererBuilder); + } + + private class RendererContext implements CommonMarkNodeRendererContext { + private final CommonMarkWriter commonMarkWriter; + private final NodeRendererMap nodeRendererMap = new NodeRendererMap(); + + private RendererContext(CommonMarkWriter commonMarkWriter) { + this.commonMarkWriter = commonMarkWriter; + + // The first node renderer for a node type "wins". + for (int i = nodeRendererFactories.size() - 1; i >= 0; i--) { + CommonmarkNodeRendererFactory nodeRendererFactory = nodeRendererFactories.get(i); + NodeRenderer nodeRenderer = nodeRendererFactory.create(this); + nodeRendererMap.add(nodeRenderer); + } + } + + @Override + public boolean stripNewlines() { + return stripNewlines; + } + + @Override + public CommonMarkWriter getWriter() { + return commonMarkWriter; + } + + @Override + public void render(Node node) { + nodeRendererMap.render(node); + } + } +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java new file mode 100644 index 000000000..9d0fdbbd8 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java @@ -0,0 +1,65 @@ +package org.commonmark.renderer.roundtrip; + +import java.io.IOException; + +public class CommonMarkWriter { + + private final Appendable buffer; + + private char lastChar; + + public CommonMarkWriter(Appendable out) { + buffer = out; + } + + public void whitespace() { + if (lastChar != 0 && lastChar != ' ') { + append(' '); + } + } + + public void colon() { + if (lastChar != 0 && lastChar != ':') { + append(':'); + } + } + + public void line() { + append('\n'); + } + + public void writeStripped(String s) { + append(s.replaceAll("[\\r\\n\\s]+", " ")); + } + + public void write(String s) { + append(s); + } + + public void write(char c) { + append(c); + } + + private void append(String s) { + try { + buffer.append(s); + } catch (IOException e) { + throw new RuntimeException(e); + } + + int length = s.length(); + if (length != 0) { + lastChar = s.charAt(length - 1); + } + } + + private void append(char c) { + try { + buffer.append(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + + lastChar = c; + } +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java new file mode 100644 index 000000000..9c3bd61f4 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java @@ -0,0 +1,17 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.renderer.NodeRenderer; + +/** + * Factory for instantiating new node renderers when rendering is done. + */ +public interface CommonmarkNodeRendererFactory { + + /** + * Create a new node renderer for the specified rendering context. + * + * @param context the context for rendering (normally passed on to the node renderer) + * @return a node renderer + */ + NodeRenderer create(CommonMarkNodeRendererContext context); +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java new file mode 100644 index 000000000..2f9707f05 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java @@ -0,0 +1,561 @@ +package org.commonmark.renderer.roundtrip; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.commonmark.internal.renderer.text.BulletListHolder; +import org.commonmark.internal.renderer.text.ListHolder; +import org.commonmark.internal.renderer.text.OrderedListHolder; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.BlankLine; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Document; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.renderer.NodeRenderer; + +/** + * The node renderer that renders all the core nodes (comes last in the order of node renderers). + */ +public class CoreCommonMarkNodeRenderer extends AbstractVisitor implements NodeRenderer { + + protected final CommonMarkNodeRendererContext context; + private final CommonMarkWriter textContent; + + private ListHolder listHolder; + + public CoreCommonMarkNodeRenderer(CommonMarkNodeRendererContext context) { + this.context = context; + this.textContent = context.getWriter(); + } + + @Override + public Set> getNodeTypes() { + return new HashSet<>(Arrays.asList( + Document.class, + Heading.class, + Paragraph.class, + BlockQuote.class, + BulletList.class, + FencedCodeBlock.class, + HtmlBlock.class, + ThematicBreak.class, + IndentedCodeBlock.class, + Link.class, + LinkReferenceDefinition.class, + ListBlock.class, + ListItem.class, + OrderedList.class, + Image.class, + Emphasis.class, + StrongEmphasis.class, + Text.class, + Code.class, + HtmlInline.class, + SoftLineBreak.class, + HardLineBreak.class, + BlankLine.class + )); + } + + @Override + public void render(Node node) { + node.accept(this); + } + + @Override + public void visit(Document document) { + // No rendering itself (aside from post-block whitespace, if present) + + visitChildren(document); + + textContent.write(document.whitespaceEndOfDocument()); + } + + public void visit(BlockQuote blockQuote) { + if(blockQuote.getFirstChild() instanceof Heading || + blockQuote.getFirstChild() instanceof IndentedCodeBlock || + blockQuote.getFirstChild() instanceof FencedCodeBlock) { + textContent.write(blockQuote.whitespacePreMarker()); + textContent.write(">"); + textContent.write(blockQuote.whitespacePostMarker()); + } + + visitChildren(blockQuote); + + writeEndOfLineIfNeeded(blockQuote, null); + } + + @Override + public void visit(BulletList bulletList) { + listHolder = new BulletListHolder(listHolder, bulletList); + visitChildren(bulletList); + writeEndOfLineIfNeeded(bulletList, null); + + if (listHolder.getParent() != null) { + listHolder = listHolder.getParent(); + } else { + listHolder = null; + } + } + + public void visit(Code code) { + for(int i = 0; i < code.getNumBackticks(); i++) { + textContent.write('`'); + } + + textContent.write(code.getRaw()); + + for(int i = 0; i < code.getNumBackticks(); i++) { + textContent.write('`'); + } + } + + @Override + public void visit(Emphasis emphasis) { + textContent.write(emphasis.whitespacePreBlock()); + textContent.write(emphasis.getOpeningDelimiter()); + visitChildren(emphasis); + textContent.write(emphasis.getClosingDelimiter()); + } + + public void visit(FencedCodeBlock fencedCodeBlock) { + for(int i = 0; i < fencedCodeBlock.getStartFenceIndent(); i++) { + textContent.write(" "); + } + + for(int i = 0; i < fencedCodeBlock.getStartFenceLength(); i++) { + textContent.write(fencedCodeBlock.getFenceChar()); + } + + if(fencedCodeBlock.getInfo() != null && !fencedCodeBlock.getInfo().isEmpty()) { + textContent.write(fencedCodeBlock.getInfo()); + } + + // CommonMark test case #96 shows that content with no ending fence and + // completely blank content should not trigger a new line + if(fencedCodeBlock.getEndFenceLength() > 0 || !fencedCodeBlock.getRaw().isEmpty()) { + textContent.line(); + } + + textContent.write(fencedCodeBlock.getRaw()); + + if(fencedCodeBlock.getEndFenceLength() > 0 && !fencedCodeBlock.getRaw().isEmpty()) { + textContent.line(); + } + + for(int i = 0; i < fencedCodeBlock.getEndFenceIndent(); i++) { + textContent.write(" "); + } + + for(int i = 0; i < fencedCodeBlock.getEndFenceLength(); i++) { + textContent.write(fencedCodeBlock.getFenceChar()); + } + + if(fencedCodeBlock.getEndFenceLength() > 0) { + textContent.write(fencedCodeBlock.whitespacePostBlock()); + } + + writeEndOfLineIfNeeded(fencedCodeBlock, null); + } + + @Override + public void visit(HardLineBreak hardLineBreak) { + if(hardLineBreak.hasBackslash()) { + textContent.write("\\"); + } + + writeEndOfLine(); + } + + public void visit(Heading heading) { + + if(heading.getSymbolType() == '#') { + textContent.write(heading.whitespacePreBlock()); + + for(int i = 0; i < heading.getLevel(); i++) { + textContent.write(heading.getSymbolType()); + } + + textContent.write(heading.whitespacePreContent()); + + visitChildren(heading); + + textContent.write(heading.whitespacePostContent()); + + if(heading.getNumEndingSymbol() > 0) { + for(int i = 0; i < heading.getNumEndingSymbol(); i++) { + textContent.write(heading.getSymbolType()); + } + } + + textContent.write(heading.whitespacePostBlock()); + + writeEndOfLineIfNeeded(heading, null); + + }else { + textContent.write(heading.whitespacePreBlock()); + + visitChildren(heading); + + textContent.write(heading.whitespacePreContent()); + + writeEndOfLine(); + + textContent.write(heading.whitespacePostContent()); + + if(heading.getNumEndingSymbol() > 0) { + for(int i = 0; i < heading.getNumEndingSymbol(); i++) { + textContent.write(heading.getSymbolType()); + } + } + + textContent.write(heading.whitespacePostBlock()); + + writeEndOfLineIfNeeded(heading, null); + } + } + + public void visit(ThematicBreak thematicBreak) { + if(thematicBreak.getContent() != null) { + textContent.write(thematicBreak.getContent().toString()); + } + + writeEndOfLineIfNeeded(thematicBreak, null); + } + + @Override + public void visit(HtmlInline htmlInline) { + textContent.write(htmlInline.getLiteral()); + } + + @Override + public void visit(HtmlBlock htmlBlock) { + textContent.write(htmlBlock.getRaw()); + writeEndOfLineIfNeeded(htmlBlock, null); + } + + @Override + public void visit(Image image) { + writeLink(image, image.getTitle(), image.getDestination()); + } + + @Override + public void visit(IndentedCodeBlock indentedCodeBlock) { + textContent.write(indentedCodeBlock.getIndentWhitespace()); + + textContent.write(indentedCodeBlock.getRaw()); + + writeEndOfLineIfNeeded(indentedCodeBlock, null); + } + + public void visit(Link link) { + writeLink(link, link.getTitle(), link.getDestination()); + } + + @Override + public void visit(LinkReferenceDefinition linkDef) { + writeLinkReference(linkDef); + writeEndOfLineIfNeeded(linkDef, null); + } + + @Override + public void visit(ListItem listItem) { + if (listHolder != null && listHolder instanceof OrderedListHolder) { + OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder; + + textContent.write(listItem.whitespacePreMarker()); + + if(!(listItem.getFirstChild() instanceof ThematicBreak) && + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(listItem.getRawNumber() + ""); + textContent.write(orderedListHolder.getDelimiter()); + textContent.write(listItem.whitespacePostMarker()); + } + + visitChildren(listItem); + + writeEndOfLineIfNeeded(listItem, null); + } else if (listHolder != null && listHolder instanceof BulletListHolder) { + BulletListHolder bulletListHolder = (BulletListHolder) listHolder; + + textContent.write(listItem.whitespacePreMarker()); + + if(!(listItem.getFirstChild() instanceof ThematicBreak) && + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(bulletListHolder.getMarker()); + textContent.write(listItem.whitespacePostMarker()); + } + visitChildren(listItem); + + writeEndOfLineIfNeeded(listItem, null); + } + + } + + @Override + public void visit(OrderedList orderedList) { + listHolder = new OrderedListHolder(listHolder, orderedList); + visitChildren(orderedList); + writeEndOfLineIfNeeded(orderedList, null); + + if (listHolder.getParent() != null) { + listHolder = listHolder.getParent(); + } else { + listHolder = null; + } + + } + + public void visit(Paragraph paragraph) { + visitChildren(paragraph); + + writeEndOfLineIfNeeded(paragraph, null); + } + + @Override + public void visit(SoftLineBreak softLineBreak) { + // Within roundtrip rendering, a soft break is always a newline of some kind + writeEndOfLine(); + } + + @Override + public void visit(StrongEmphasis strongEmphasis) { + textContent.write(strongEmphasis.whitespacePreBlock()); + textContent.write(strongEmphasis.getOpeningDelimiter()); + visitChildren(strongEmphasis); + textContent.write(strongEmphasis.getClosingDelimiter()); + } + + @Override + public void visit(Text text) { + if(!text.getRaw().isEmpty()) { + textContent.write(text.whitespacePreContent() + text.getRaw() + text.whitespacePostContent()); + }else { + textContent.write(text.whitespacePreContent() + text.getLiteral() + text.whitespacePostContent()); + } + } + + @Override + public void visit(BlankLine blankLine) { + System.out.println(blankLine.getRaw()); + textContent.write(blankLine.getRaw()); + writeEndOfLineIfNeeded(blankLine, null); + } + + @Override + protected void visitChildren(Node parent) { + Node node = parent.getFirstChild(); + + while (node != null) { + Node next = node.getNext(); + + context.render(node); + node = next; + } + } + + private void writeLink(Node node, String title, String destination) { + if(node instanceof Link) { + writeLink((Link)node); + } + + if(node instanceof Image) { + writeImage((Image)node); + } + } + + private void writeLink(Link node) { + // Autolink + if(node.getLinkType() == LinkType.AUTOLINK) { + textContent.write("<"); + visitChildren(node); + textContent.write(">"); + }else if(node.getLinkType() == LinkType.REFERENCE) { + textContent.write("["); + visitChildren(node); + textContent.write("]"); + + if(node.getLabel() != null) { + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); + } + }else { + textContent.write("["); + visitChildren(node); + textContent.write("]"); + + textContent.write("("); + + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); + }else if(node.getDestination() != null) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getDestination()); + }else { + visitChildren(node); + } + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); + }else if(node.getTitle() != null) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getTitleSymbol() + ""); + + textContent.write(node.getTitle()); + + if(node.getTitleSymbol() == '(') { + textContent.write(")"); + }else { + textContent.write(node.getTitleSymbol()); + } + } + + textContent.write(node.getWhitespacePostContent()); + textContent.write(")"); + } + } + + private void writeLinkReference(LinkReferenceDefinition node) { + textContent.write(node.whitespacePreLabel()); + + textContent.write("["); + textContent.write(node.getLabel()); + textContent.write("]:"); + + if(node.getDestination() != null) { + textContent.write(node.whitespacePreDestination()); + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getRawDestination()); + }else { + textContent.write(node.getDestination()); + } + + if(node.getTitle() != null) { + textContent.write(node.whitespacePreTitle()); + + char delimiter = node.getDelimiterChar(); + + if(delimiter == Character.MIN_VALUE) { + delimiter = '"'; + } + + textContent.write(delimiter); + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getRawTitle()); + }else { + textContent.write(node.getTitle()); + } + + textContent.write(delimiter); + } + } + + textContent.write(node.whitespacePostTitle()); + } + + private void writeImage(Image node) { + // Autolink + if(node.getLinkType() == LinkType.AUTOLINK) { + textContent.write("<"); + visitChildren(node); + textContent.write(">"); + }else if(node.getLinkType() == LinkType.REFERENCE) { + textContent.write("!["); + visitChildren(node); + textContent.write("]"); + + if(node.getLabel() != null) { + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); + } + }else { + textContent.write("!["); + visitChildren(node); + textContent.write("]"); + + textContent.write("("); + + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); + }else if(node.getDestination() != null) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getDestination()); + }else { + visitChildren(node); + } + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); + }else if(node.getTitle() != null) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getTitleSymbol() + ""); + + textContent.write(node.getTitle()); + + if(node.getTitleSymbol() == '(') { + textContent.write(")"); + }else { + textContent.write(node.getTitleSymbol()); + } + } + + textContent.write(node.getWhitespacePostContent()); + textContent.write(")"); + } + } + + private void writeEndOfLineIfNeeded(Node node, Character c) { + if (node.getNext() != null) { + if(node instanceof ListItem || node instanceof Paragraph) { + textContent.line(); + }else { + textContent.line(); + } + } + } + + private void writeEndOfLine() { + textContent.line(); + } +} diff --git a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java index 173d711d3..7bcc4a963 100644 --- a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java +++ b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java @@ -102,6 +102,11 @@ public int getMinLength() { public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return delimiterUse; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class AsymmetricDelimiterProcessor implements DelimiterProcessor { @@ -136,6 +141,11 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class UpperCaseNode extends CustomNode { @@ -198,6 +208,11 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { closingRun.getCloser().insertBefore(new Text("(/1)")); return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class TwoDelimiterProcessor implements DelimiterProcessor { @@ -223,5 +238,10 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { closingRun.getCloser().insertBefore(new Text("(/2)")); return 2; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } } diff --git a/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java b/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java index bf7bde6ec..b0f1427d6 100644 --- a/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java +++ b/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java @@ -17,9 +17,11 @@ public void testDefinitionWithoutParagraph() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url 'title'"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(2)); + // Node size is 3 instead of 2 due to BlankLine pseudo-node + assertThat(nodes.size(), is(3)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - LinkReferenceDefinition definition = assertDef(nodes.get(1), "foo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + LinkReferenceDefinition definition = assertDef(nodes.get(2), "foo"); assertThat(definition.getDestination(), is("/url")); assertThat(definition.getTitle(), is("title")); @@ -41,10 +43,12 @@ public void testMultipleDefinitions() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url\n[bar]: /url"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(3)); + // Node size is 4 instead of 3 due to BlankLine pseudo-node + assertThat(nodes.size(), is(4)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - assertDef(nodes.get(1), "foo"); - assertDef(nodes.get(2), "bar"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + assertDef(nodes.get(2), "foo"); + assertDef(nodes.get(3), "bar"); } @Test @@ -52,13 +56,15 @@ public void testMultipleDefinitionsWithSameLabel() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url1\n[foo]: /url2"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(3)); + // Node size is 4 instead of 3 due to BlankLine pseudo-node + assertThat(nodes.size(), is(4)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - LinkReferenceDefinition def1 = assertDef(nodes.get(1), "foo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + LinkReferenceDefinition def1 = assertDef(nodes.get(2), "foo"); assertThat(def1.getDestination(), is("/url1")); // When there's multiple definitions with the same label, the first one "wins", as in reference links will use // that. But we still want to preserve the original definitions in the document. - LinkReferenceDefinition def2 = assertDef(nodes.get(2), "foo"); + LinkReferenceDefinition def2 = assertDef(nodes.get(3), "foo"); assertThat(def2.getDestination(), is("/url2")); } @@ -110,9 +116,11 @@ public void testDefinitionLabelCaseIsPreserved() { Node document = parse("This is a paragraph with a [foo] link.\n\n[fOo]: /url 'title'"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(2)); + // Node size is 3 instead of 2 due to BlankLine pseudo-node + assertThat(nodes.size(), is(3)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - assertDef(nodes.get(1), "fOo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + assertDef(nodes.get(2), "fOo"); } private static Node parse(String input) {