From b46c7f1f79e5321b9d6ad98e424ffb02dec78847 Mon Sep 17 00:00:00 2001 From: Seii Date: Wed, 25 Aug 2021 11:34:30 -0600 Subject: [PATCH 1/5] Add roundtrip rendering feature --- .../commonmark/internal/BlockQuoteParser.java | 26 +- .../commonmark/internal/DocumentParser.java | 105 +++- .../internal/FencedCodeBlockParser.java | 83 ++- .../commonmark/internal/HeadingParser.java | 119 +++- .../commonmark/internal/HtmlBlockParser.java | 31 +- .../internal/IndentedCodeBlockParser.java | 63 +- .../commonmark/internal/InlineParserImpl.java | 281 ++++++++- .../LinkReferenceDefinitionParser.java | 102 +++- .../commonmark/internal/ListBlockParser.java | 76 ++- .../commonmark/internal/ListItemParser.java | 20 +- .../commonmark/internal/ParagraphParser.java | 43 +- .../internal/StaggeredDelimiterProcessor.java | 6 + .../internal/ThematicBreakParser.java | 29 +- .../internal/inline/AutolinkInlineParser.java | 12 +- .../inline/BackslashInlineParser.java | 10 +- .../inline/BackticksInlineParser.java | 19 +- .../internal/inline/BlankLineParser.java | 28 + .../inline/EmphasisDelimiterProcessor.java | 45 ++ .../internal/inline/EntityInlineParser.java | 8 +- .../internal/inline/HtmlInlineParser.java | 6 + .../internal/inline/InlineContentParser.java | 2 + .../commonmark/internal/inline/Scanner.java | 53 +- .../renderer/text/OrderedListHolder.java | 6 + .../commonmark/internal/util/LinkScanner.java | 13 +- .../org/commonmark/internal/util/Parsing.java | 50 ++ .../org/commonmark/node/AbstractVisitor.java | 5 + .../java/org/commonmark/node/BlankLine.java | 52 ++ .../main/java/org/commonmark/node/Block.java | 36 ++ .../java/org/commonmark/node/BlockQuote.java | 32 +- .../java/org/commonmark/node/BulletList.java | 29 + .../main/java/org/commonmark/node/Code.java | 27 + .../java/org/commonmark/node/Document.java | 30 + .../java/org/commonmark/node/Emphasis.java | 10 + .../org/commonmark/node/FencedCodeBlock.java | 67 +- .../org/commonmark/node/HardLineBreak.java | 16 +- .../java/org/commonmark/node/Heading.java | 48 ++ .../java/org/commonmark/node/HtmlBlock.java | 39 ++ .../java/org/commonmark/node/HtmlInline.java | 9 + .../main/java/org/commonmark/node/Image.java | 132 +++- .../commonmark/node/IndentedCodeBlock.java | 44 ++ .../main/java/org/commonmark/node/Link.java | 132 +++- .../java/org/commonmark/node/LinkFormat.java | 30 + .../node/LinkReferenceDefinition.java | 56 +- .../java/org/commonmark/node/ListItem.java | 39 ++ .../main/java/org/commonmark/node/Node.java | 15 + .../java/org/commonmark/node/OrderedList.java | 38 ++ .../java/org/commonmark/node/Paragraph.java | 30 + .../org/commonmark/node/StrongEmphasis.java | 12 + .../main/java/org/commonmark/node/Text.java | 64 +- .../org/commonmark/node/ThematicBreak.java | 40 ++ .../java/org/commonmark/node/Visitor.java | 2 + .../org/commonmark/parser/SourceLine.java | 59 ++ .../org/commonmark/parser/SourceLines.java | 33 +- .../parser/delimiter/DelimiterProcessor.java | 15 + .../renderer/html/CoreHtmlNodeRenderer.java | 51 +- .../CommonMarkNodeRendererContext.java | 25 + .../roundtrip/CommonMarkRenderer.java | 149 +++++ .../renderer/roundtrip/CommonMarkWriter.java | 65 ++ .../CommonmarkNodeRendererFactory.java | 17 + .../roundtrip/CoreCommonMarkNodeRenderer.java | 575 ++++++++++++++++++ .../test/DelimiterProcessorTest.java | 20 + .../test/LinkReferenceDefinitionNodeTest.java | 28 +- .../java/org/commonmark/test/ParserTest.java | 25 + 63 files changed, 3180 insertions(+), 152 deletions(-) create mode 100644 commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java create mode 100644 commonmark/src/main/java/org/commonmark/node/BlankLine.java create mode 100644 commonmark/src/main/java/org/commonmark/node/LinkFormat.java create mode 100644 commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java create mode 100644 commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java create mode 100644 commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java create mode 100644 commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java create mode 100644 commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java diff --git a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java index 00cdbc11e..87f016689 100644 --- a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java @@ -9,6 +9,15 @@ public class BlockQuoteParser extends AbstractBlockParser { private final BlockQuote block = new BlockQuote(); + // Preserve original default constructor by explicitly defining one + public BlockQuoteParser() { + super(); + } + + public BlockQuoteParser(String preBlockWhitespace, String postMarkerWhitespace) { + block.setWhitespace(preBlockWhitespace, postMarkerWhitespace); + } + @Override public boolean isContainer() { return true; @@ -47,13 +56,28 @@ private static boolean isMarker(ParserState state, int index) { public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { int nextNonSpace = state.getNextNonSpaceIndex(); + String preBlockWhitespace = ""; + + if(nextNonSpace > 0) { + if(state.getLine().getContent().subSequence(0, nextNonSpace).toString().isBlank()) { + preBlockWhitespace = state.getLine().getContent().subSequence(0, nextNonSpace).toString(); + }else { + preBlockWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace, 0); + } + } + if (isMarker(state, nextNonSpace)) { int newColumn = state.getColumn() + state.getIndent() + 1; + // optional following space or tab + String optionalWhitespace = ""; + if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) { + optionalWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), nextNonSpace + 1, state.getLine().getContent().length() - 1); newColumn++; } - return BlockStart.of(new BlockQuoteParser()).atColumn(newColumn); + + return BlockStart.of(new BlockQuoteParser(preBlockWhitespace, optionalWhitespace)).atColumn(newColumn); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java index ed8ae7412..2506779cd 100644 --- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java @@ -1,15 +1,43 @@ package org.commonmark.internal; -import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; -import org.commonmark.parser.*; -import org.commonmark.parser.block.*; -import org.commonmark.parser.delimiter.DelimiterProcessor; - import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.inline.BlankLineParser; +import org.commonmark.internal.util.Parsing; +import org.commonmark.node.Block; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.Document; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.ListBlock; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SourceSpan; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.IncludeSourceSpans; +import org.commonmark.parser.InlineParser; +import org.commonmark.parser.InlineParserFactory; +import org.commonmark.parser.SourceLine; +import org.commonmark.parser.SourceLines; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockParser; +import org.commonmark.parser.block.BlockParserFactory; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; +import org.commonmark.parser.delimiter.DelimiterProcessor; public class DocumentParser implements ParserState { @@ -275,13 +303,19 @@ private void parseLine(CharSequence ln) { addLine(); } else { - // finalize any blocks not matched if (unmatchedBlocks > 0) { closeBlockParsers(unmatchedBlocks); } if (!blockParser.isContainer()) { + // Capture any blank lines within a separate node for roundtrip purposes + // BlankLine nodes do _not_ interrupt the current node type's processor + if(isBlank()) { + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } + addLine(); } else if (!isBlank()) { // create paragraph container for line @@ -297,6 +331,13 @@ private void parseLine(CharSequence ln) { // // The first line does not start a paragraph yet, but we still want to record source positions. addSourceSpans(); + + // Capture any blank lines within a separate node for roundtrip purposes + // BlankLine nodes do _not_ interrupt the current node type's processor + if(isBlank()) { + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } } } } @@ -403,18 +444,23 @@ private void addLine() { } sb.append(rest); content = sb.toString(); - } else if (index == 0) { - content = line.getContent(); } else { - content = line.getContent().subSequence(index, line.getContent().length()); + content = line.getContent().toString(); } + SourceSpan sourceSpan = null; if (includeSourceSpans == IncludeSourceSpans.BLOCKS_AND_INLINES) { // Note that if we're in a partially-consumed tab, the length here corresponds to the content but not to the // actual source length. That sounds like a problem, but I haven't found a test case where it matters (yet). - sourceSpan = SourceSpan.of(lineIndex, index, content.length()); + sourceSpan = SourceSpan.of(lineIndex, index, line.getLiteralLine().getContent().length()); + } + + if(!columnIsInTab) { + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, index)); + }else { + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, 0)); } - getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan)); + addSourceSpans(); } @@ -425,6 +471,11 @@ private void addSourceSpans() { OpenBlockParser openBlockParser = openBlockParsers.get(i); int blockIndex = openBlockParser.sourceIndex; int length = line.getContent().length() - blockIndex; + + if(line.getLiteralIndex() != 0) { + length = line.getLiteralLine().getContent().length() - blockIndex; + } + if (length != 0) { openBlockParser.blockParser.addSourceSpan(SourceSpan.of(lineIndex, blockIndex, length)); } @@ -481,12 +532,21 @@ private void processInlines() { * its parent, and so on until we find a block that can accept children. */ private void addChild(OpenBlockParser openBlockParser) { - while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { - closeBlockParsers(1); - } + // BlankLine nodes do _not_ interrupt the current node type's processor, as + // they are non-standard CommonMark nodes meant only for roundtrip processing + if(!(openBlockParser.blockParser instanceof BlankLineParser)) { + while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { + closeBlockParsers(1); + } + } getActiveBlockParser().getBlock().appendChild(openBlockParser.blockParser.getBlock()); - activateBlockParser(openBlockParser); + + // BlankLine nodes do _not_ become the active parser, as this would interrupt the + // parser before it + if(!(openBlockParser.blockParser instanceof BlankLineParser)) { + activateBlockParser(openBlockParser); + } } private void activateBlockParser(OpenBlockParser openBlockParser) { @@ -520,6 +580,17 @@ private Block prepareActiveBlockParserForReplacement() { private Document finalizeAndProcess() { closeBlockParsers(openBlockParsers.size()); processInlines(); + Document docNode = documentBlockParser.getBlock(); + String postBlockWhitespace = ""; + + // CommonMark test cases end with newlines, indicating that this should be standard + // in rountrip scenarios. If a post-document whitespace doesn't exist, add it. + if(postBlockWhitespace.isEmpty()) { + postBlockWhitespace = "\n"; + } + + docNode.setWhitespace(docNode.whitespacePreBlock(), docNode.whitespacePreContent(), + docNode.whitespacePostContent(), postBlockWhitespace); return documentBlockParser.getBlock(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java index 2d7d2c0c9..074c6d9b1 100644 --- a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java @@ -4,9 +4,12 @@ import org.commonmark.node.Block; import org.commonmark.node.FencedCodeBlock; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import static org.commonmark.internal.util.Escaping.unescapeString; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class FencedCodeBlockParser extends AbstractBlockParser { @@ -14,11 +17,15 @@ public class FencedCodeBlockParser extends AbstractBlockParser { private String firstLine; private StringBuilder otherLines = new StringBuilder(); + private StringBuilder rawLines = new StringBuilder(); public FencedCodeBlockParser(char fenceChar, int fenceLength, int fenceIndent) { block.setFenceChar(fenceChar); - block.setFenceLength(fenceLength); - block.setFenceIndent(fenceIndent); + block.setStartFenceLength(fenceLength); + + // Fenced code blocks can't be indented by newlines or tabs, so it's safe to assume + // spaces for the indentation + block.setWhitespace(Parsing.generateSpaces(fenceIndent)); } @Override @@ -32,11 +39,20 @@ public BlockContinue tryContinue(ParserState state) { int newIndex = state.getIndex(); CharSequence line = state.getLine().getContent(); if (state.getIndent() < Parsing.CODE_BLOCK_INDENT && nextNonSpace < line.length() && line.charAt(nextNonSpace) == block.getFenceChar() && isClosing(line, nextNonSpace)) { - // closing fence - we're at end of line, so we can finalize now + // Capture whitespace before closing fence (if any) for roundtrip purposes + if(nextNonSpace > 0) { + block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), Parsing.collectWhitespace(line, 0, nextNonSpace), block.whitespacePostBlock()); + } + + // closing fence - we're at end of line, so we can finalize now return BlockContinue.finished(); } else { + // Capture line before optional spaces are removed + rawLines.append(line.toString()); + rawLines.append('\n'); + // skip optional spaces of fence indent - int i = block.getFenceIndent(); + int i = block.getStartFenceIndent(); int length = line.length(); while (i > 0 && newIndex < length && line.charAt(newIndex) == ' ') { newIndex++; @@ -49,9 +65,24 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { if (firstLine == null) { - firstLine = line.getContent().toString(); + + // Raw line is the same as literal line, so just use it + if(line.getLiteralIndex() == 0) { + firstLine = line.getContent().toString(); + } + // Raw line differs from literal line, so preserve literal line + else { + firstLine = line.substring(line.getLiteralIndex(), line.getContent().length()).getContent().toString(); + } } else { - otherLines.append(line.getContent()); + // Raw line is the same as literal line, so just use it + if(line.getLiteralIndex() == 0) { + otherLines.append(line.getContent()); + } + // Raw line differs from literal line, so preserve literal line + else { + otherLines.append(line.getLiteralLine().getContent()); + } otherLines.append('\n'); } } @@ -59,7 +90,22 @@ public void addLine(SourceLine line) { @Override public void closeBlock() { // first line becomes info string - block.setInfo(unescapeString(firstLine.trim())); + // It is not trimmed or escaped here to allow roundtrip processing + block.setInfo(firstLine); + + // Remove final newline in raw content, if it exists + if(rawLines != null && rawLines.length() != 0) { + if(rawLines.charAt(rawLines.length() - 1) == '\n') { + rawLines.deleteCharAt(rawLines.length() - 1); + } + } + + // Capture raw content (no indentation removed) between end of info string + // and beginning of end fence + block.setRaw(rawLines.toString()); + + // Capture content (without indentation) between end of info string + // and beginning of end fence block.setLiteral(otherLines.toString()); } @@ -72,10 +118,14 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar return BlockStart.none(); } + if(indent == 0) { + indent = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()).length(); + } + int nextNonSpace = state.getNextNonSpaceIndex(); FencedCodeBlockParser blockParser = checkOpener(state.getLine().getContent(), nextNonSpace, indent); if (blockParser != null) { - return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getFenceLength()); + return BlockStart.of(blockParser).atIndex(nextNonSpace + blockParser.block.getStartFenceLength()); } else { return BlockStart.none(); } @@ -120,13 +170,22 @@ private static FencedCodeBlockParser checkOpener(CharSequence line, int index, i // code fence. private boolean isClosing(CharSequence line, int index) { char fenceChar = block.getFenceChar(); - int fenceLength = block.getFenceLength(); + int fenceLength = block.getStartFenceLength(); int fences = Parsing.skip(fenceChar, line, index, line.length()) - index; if (fences < fenceLength) { return false; + }else { + block.setEndFenceLength(fences); } + // spec: The closing code fence [...] may be followed only by spaces, which are ignored. int after = Parsing.skipSpaceTab(line, index + fences, line.length()); + + // Capture post-fence spaces (if any) for roundtrip purposes + if(after > 0 && after != (index + fences)) { + block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), block.whitespacePostContent(), Parsing.collectWhitespaceBackwards(line, line.length() - 1, 0)); + } + return after == line.length(); } } diff --git a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java index 81c60d0d1..702300db5 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java @@ -8,7 +8,12 @@ import org.commonmark.parser.InlineParser; import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import org.commonmark.parser.block.*; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class HeadingParser extends AbstractBlockParser { @@ -16,9 +21,28 @@ public class HeadingParser extends AbstractBlockParser { private final SourceLines content; public HeadingParser(int level, SourceLines content) { - block.setLevel(level); + block.setLevel(level); + block.setSymbolType('#'); this.content = content; } + + public HeadingParser(int level, SourceLines content, char symbolType, int numEndingSymbols, String... whitespace) { + this(level, content); + block.setSymbolType(symbolType); + block.setNumEndingSymbol(numEndingSymbols); + + if(whitespace.length == 4) { + block.setWhitespace(whitespace); + }else { + String[] tempArray = {"", "", "", ""}; + + for(int i = 0; i < whitespace.length; i++) { + tempArray[i] = whitespace[i]; + } + + block.setWhitespace(tempArray); + } + } @Override public Block getBlock() { @@ -46,8 +70,24 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar SourceLine line = state.getLine(); int nextNonSpace = state.getNextNonSpaceIndex(); + + String preBlockWhitespace = ""; + if (line.getContent().charAt(nextNonSpace) == '#') { - HeadingParser atxHeading = getAtxHeading(line.substring(nextNonSpace, line.getContent().length())); + + // Account for block-in-a-block instances during roundtrips, like Heading inside a List + if(nextNonSpace > 0) { + preBlockWhitespace = Parsing.collectWhitespaceBackwards(line.getContent(), nextNonSpace - 1, 0); + + // If these values don't match, it's a block-in-a-block scenario because + // there's another character before the whitespace + if(preBlockWhitespace.length() != nextNonSpace) { + preBlockWhitespace = ""; + } + } + + HeadingParser atxHeading = getAtxHeading(line.substring(nextNonSpace - preBlockWhitespace.length(), line.getContent().length()), nextNonSpace); + if (atxHeading != null) { return BlockStart.of(atxHeading).atIndex(line.getContent().length()); } @@ -55,9 +95,46 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar int setextHeadingLevel = getSetextHeadingLevel(line.getContent(), nextNonSpace); if (setextHeadingLevel > 0) { + int numEndingSymbols = line.getContent().toString().trim().length(); + + // AST: Setext headings have a slightly different twist on whitespace: + // Pre-block = Before setext text begins + // Pre-content = Directly after setext text + // Post-content = Before setext delimiting line + // Post-block = After setext delimiting line + String preContentWhitespace = ""; + String postContentWhitespace = Parsing.collectWhitespace(line.getContent(), 0, line.getContent().length()); + String postBlockWhitespace = Parsing.collectWhitespaceBackwards(line.getContent(), line.getContent().length() - 1, 0); + + // If the setext had valid content, the current matched block + // parser will be the paragraph parser. Setext headings use + // Text (not Paragraph) in the final output, but it's still + // possible to gather the right whitespace from Paragraph. + boolean isMatchedParagraph = matchedBlockParser.getMatchedBlockParser() instanceof ParagraphParser; + + if(isMatchedParagraph) { + SourceLines rawLines = ((ParagraphParser)matchedBlockParser.getMatchedBlockParser()).getRawParagraphLines(); + + // In some instances, the paragraph object exists but has not yet been populated. + // However, the link reference definition parser will still have the paragraph's + // lines ready for capture. If no whitespace is available yet, capture any initial + // whitespace in the first paragraph line. + if(preBlockWhitespace.isEmpty() && !rawLines.isEmpty()) { + CharSequence rawParagraphSequence = rawLines.getLines().get(0).getContent(); + preBlockWhitespace = Parsing.collectWhitespace(rawParagraphSequence.toString(), 0, rawParagraphSequence.length()); + } + } + + char symbolType; + if(setextHeadingLevel == 1) { + symbolType = '='; + }else { + symbolType = '-'; + } + SourceLines paragraph = matchedBlockParser.getParagraphLines(); if (!paragraph.isEmpty()) { - return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph)) + return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph, symbolType, numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace, postBlockWhitespace)) .atIndex(line.getContent().length()) .replaceActiveBlockParser(); } @@ -72,7 +149,14 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar // sequence of # characters must be followed by a space or by the end of line. The optional closing sequence of #s // must be preceded by a space and may be followed by spaces only. private static HeadingParser getAtxHeading(SourceLine line) { + return getAtxHeading(line, 0); + } + + private static HeadingParser getAtxHeading(SourceLine line, int nextNonSpace) { + String preBlockWhitespace = ""; + Scanner scanner = Scanner.of(SourceLines.of(line)); + preBlockWhitespace = scanner.whitespaceAsString(); int level = scanner.matchMultiple('#'); if (level == 0 || level > 6) { @@ -89,23 +173,28 @@ private static HeadingParser getAtxHeading(SourceLine line) { return null; } - scanner.whitespace(); + String preContentWhitespace = scanner.whitespaceAsString(); Position start = scanner.position(); Position end = start; boolean hashCanEnd = true; + int numEndingSymbols = 0; + StringBuilder postContentWhitespace = new StringBuilder(); + String postBlockWhitespace = ""; while (scanner.hasNext()) { char c = scanner.peek(); switch (c) { case '#': if (hashCanEnd) { - scanner.matchMultiple('#'); - int whitespace = scanner.whitespace(); + numEndingSymbols = scanner.matchMultiple('#'); + String whitespace = scanner.whitespaceAsString(); // If there's other characters, the hashes and spaces were part of the heading if (scanner.hasNext()) { + numEndingSymbols = 0; + postContentWhitespace.setLength(0); end = scanner.position(); } - hashCanEnd = whitespace > 0; + hashCanEnd = whitespace.length() > 0; } else { scanner.next(); end = scanner.position(); @@ -114,21 +203,29 @@ private static HeadingParser getAtxHeading(SourceLine line) { case ' ': case '\t': hashCanEnd = true; + postContentWhitespace.append(c); scanner.next(); break; default: hashCanEnd = false; + postContentWhitespace.setLength(0); scanner.next(); end = scanner.position(); } } + + if(numEndingSymbols > 0) { + int beforePostBlockWhitespaceIndex = Parsing.skipSpaceTabBackwards(line.getContent(), line.getContent().length() - 1, 0); + postBlockWhitespace = line.getContent().subSequence(beforePostBlockWhitespaceIndex + 1, line.getContent().length()).toString(); + } SourceLines source = scanner.getSource(start, end); String content = source.getContent(); - if (content.isEmpty()) { - return new HeadingParser(level, SourceLines.empty()); + + if(content.isEmpty()) { + return new HeadingParser(level, SourceLines.empty(), '#', numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace.toString(), postBlockWhitespace); } - return new HeadingParser(level, source); + return new HeadingParser(level, source, '#', numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace.toString(), postBlockWhitespace); } // spec: A setext heading underline is a sequence of = characters or a sequence of - characters, with no more than diff --git a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java index 0e2050567..b240681f2 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java @@ -1,13 +1,18 @@ package org.commonmark.internal; +import java.util.regex.Pattern; + import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.HtmlBlock; import org.commonmark.node.Paragraph; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import java.util.regex.Pattern; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class HtmlBlockParser extends AbstractBlockParser { @@ -64,6 +69,7 @@ public class HtmlBlockParser extends AbstractBlockParser { private boolean finished = false; private BlockContent content = new BlockContent(); + private BlockContent rawContent = new BlockContent(); private HtmlBlockParser(Pattern closingPattern) { this.closingPattern = closingPattern; @@ -90,7 +96,22 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - content.add(line.getContent()); + // Raw line and literal line are the same + if(line.getLiteralIndex() == 0) { + content.add(line.getContent()); + rawContent.add(line.getContent()); + } + // Raw line and literal line are different + else { + content.add(line.getLiteralLine().getContent()); + + // Use the literal line for anything which may be a block quote + if(line != null && line.getContent().toString().contains(">")) { + rawContent.add(line.getContent()); + }else { + rawContent.add(line.getLiteralLine().getContent()); + } + } if (closingPattern != null && closingPattern.matcher(line.getContent()).find()) { finished = true; @@ -100,7 +121,9 @@ public void addLine(SourceLine line) { @Override public void closeBlock() { block.setLiteral(content.getString()); + block.setRaw(rawContent.getString()); content = null; + rawContent = null; } public static class Factory extends AbstractBlockParserFactory { diff --git a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java index af74a587c..d4da4bff8 100644 --- a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java @@ -1,20 +1,35 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.List; + import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.IndentedCodeBlock; import org.commonmark.node.Paragraph; import org.commonmark.parser.SourceLine; -import org.commonmark.parser.block.*; - -import java.util.ArrayList; -import java.util.List; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class IndentedCodeBlockParser extends AbstractBlockParser { private final IndentedCodeBlock block = new IndentedCodeBlock(); private final List lines = new ArrayList<>(); + private final List rawLines = new ArrayList<>(); + // Preserve original default constructor by explicitly defining one + public IndentedCodeBlockParser() { + super(); + } + + public IndentedCodeBlockParser(String preBlockWhitespace, String preContentWhitespace) { + block.setWhitespace(preBlockWhitespace, preContentWhitespace); + } + @Override public Block getBlock() { return block; @@ -33,6 +48,22 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { + String prefix = ""; + + // Capture the prefix (needed for roundtrip, but unnecessary for HTML processing) + if(line.getLiteralIndex() != 0) { + prefix = line.substring(0, line.getLiteralIndex()).getContent().toString(); + line = line.getLiteralLine(); + } + + // Strip leading whitespace off first raw line (because it's already + // captured by the whitespace tracker) + if(rawLines != null && rawLines.size() == 0) { + rawLines.add(line.getContent().toString().stripLeading()); + }else { + rawLines.add(prefix + line.getContent()); + } + lines.add(line.getContent()); } @@ -52,8 +83,28 @@ public void closeBlock() { sb.append('\n'); } + // The "literal" string is optimized for HTML formatting, but omits some details + // which are important for roundtrip rendering String literal = sb.toString(); + + // Reuse the existing StringBuilder as a small optimization + sb.setLength(0); + for (int i = 0; i < rawLines.size(); i++) { + sb.append(rawLines.get(i)); + + // Don't append a newline if this block only contains one line, or to the + // last line if there is more than one line + if(rawLines.size() != 1 && i != rawLines.size() - 1) { + sb.append('\n'); + } + } + + // The "raw" string is, as much as possible, the entire raw content as it was first + // entered into the parser. This allows roundtrip parsing for indented code blocks. + String raw = sb.toString(); + block.setLiteral(literal); + block.setRaw(raw); } public static class Factory extends AbstractBlockParserFactory { @@ -62,7 +113,9 @@ public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { // An indented code block cannot interrupt a paragraph. if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT && !state.isBlank() && !(state.getActiveBlockParser().getBlock() instanceof Paragraph)) { - return BlockStart.of(new IndentedCodeBlockParser()).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); + String preContentWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()); + + return BlockStart.of(new IndentedCodeBlockParser("", preContentWhitespace)).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index c14b9e885..e43bba88b 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -1,18 +1,44 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.inline.AsteriskDelimiterProcessor; +import org.commonmark.internal.inline.AutolinkInlineParser; +import org.commonmark.internal.inline.BackslashInlineParser; +import org.commonmark.internal.inline.BackticksInlineParser; +import org.commonmark.internal.inline.EntityInlineParser; +import org.commonmark.internal.inline.HtmlInlineParser; +import org.commonmark.internal.inline.InlineContentParser; +import org.commonmark.internal.inline.InlineParserState; +import org.commonmark.internal.inline.ParsedInline; +import org.commonmark.internal.inline.ParsedInlineImpl; +import org.commonmark.internal.inline.Position; import org.commonmark.internal.inline.Scanner; -import org.commonmark.internal.inline.*; +import org.commonmark.internal.inline.UnderscoreDelimiterProcessor; import org.commonmark.internal.util.Escaping; import org.commonmark.internal.util.LinkScanner; import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Image; +import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.Node; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.SourceSpans; +import org.commonmark.node.Text; import org.commonmark.parser.InlineParser; import org.commonmark.parser.InlineParserContext; import org.commonmark.parser.SourceLines; import org.commonmark.parser.delimiter.DelimiterProcessor; -import java.util.*; - public class InlineParserImpl implements InlineParser, InlineParserState { private final BitSet specialCharacters; @@ -126,7 +152,7 @@ public void parse(SourceLines lines, Node block) { } } - processDelimiters(null); + processDelimiters(null, ""); mergeChildTextNodes(block); } @@ -138,8 +164,16 @@ void reset(SourceLines lines) { this.lastBracket = null; } + // Text node without roundtrip information private Text text(SourceLines sourceLines) { - Text text = new Text(sourceLines.getContent()); + Text text = new Text(sourceLines.getContent(), sourceLines.getContent(), "", ""); + text.setSourceSpans(sourceLines.getSourceSpans()); + return text; + } + + // Text node with roundtrip information + private Text text(SourceLines sourceLines, String preContentWhitespace, String postContentWhitespace) { + Text text = new Text(sourceLines.getContent(), sourceLines.getContent(), preContentWhitespace, postContentWhitespace); text.setSourceSpans(sourceLines.getSourceSpans()); return text; } @@ -150,6 +184,9 @@ private Text text(SourceLines sourceLines) { * On failure, return null. */ private List parseInline() { + // AST: Capture raw information needed for roundtrip rendering + String prefix = scanner.alignToLiteral(); + char c = scanner.peek(); switch (c) { @@ -158,7 +195,9 @@ private List parseInline() { case '!': return Collections.singletonList(parseBang()); case ']': - return Collections.singletonList(parseCloseBracket()); + List nodeList = Collections.singletonList(parseCloseBracket(prefix)); + prefix = ""; + return nodeList; case '\n': return Collections.singletonList(parseLineBreak()); case Scanner.END: @@ -167,14 +206,23 @@ private List parseInline() { // No inline parser, delimiter or other special handling. if (!specialCharacters.get(c)) { - return Collections.singletonList(parseText()); + List nodeList = Collections.singletonList(parseText(prefix)); + prefix = ""; + return nodeList; } List inlineParsers = this.inlineParsers.get(c); if (inlineParsers != null) { Position position = scanner.position(); for (InlineContentParser inlineParser : inlineParsers) { - ParsedInline parsedInline = inlineParser.tryParse(this); + ParsedInline parsedInline; + if(inlineParser instanceof BackticksInlineParser) { + parsedInline = inlineParser.tryParse(this, prefix); + prefix = ""; + }else { + parsedInline = inlineParser.tryParse(this); + } + if (parsedInline instanceof ParsedInlineImpl) { ParsedInlineImpl parsedInlineImpl = (ParsedInlineImpl) parsedInline; Node node = parsedInlineImpl.getNode(); @@ -192,21 +240,28 @@ private List parseInline() { DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c); if (delimiterProcessor != null) { - List nodes = parseDelimiters(delimiterProcessor, c); + List nodes = parseDelimiters(delimiterProcessor, c, prefix); + + // Preserve a prefix which has _only_ whitespace, as this may be + // needed in the following "parseText" method + if(!prefix.isEmpty() && prefix.isBlank()) { + prefix = ""; + } + if (nodes != null) { return nodes; } } // If we get here, even for a special/delimiter character, we will just treat it as text. - return Collections.singletonList(parseText()); + return Collections.singletonList(parseText(prefix)); } /** * Attempt to parse delimiters like emphasis, strong emphasis or custom delimiters. */ - private List parseDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar) { - DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar); + private List parseDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar, String prefix) { + DelimiterData res = scanDelimiters(delimiterProcessor, delimiterChar, prefix); if (res == null) { return null; } @@ -245,6 +300,7 @@ private Node parseOpenBracket() { private Node parseBang() { Position start = scanner.position(); scanner.next(); + if (scanner.next('[')) { Position contentPosition = scanner.position(); Text node = text(scanner.getSource(start, contentPosition)); @@ -262,8 +318,13 @@ private Node parseBang() { * plain [ character. If there is a matching delimiter, remove it from the delimiter stack. */ private Node parseCloseBracket() { + return parseCloseBracket(""); + } + + private Node parseCloseBracket(String prefix) { Position beforeClose = scanner.position(); scanner.next(); + Position afterClose = scanner.position(); // Get previous `[` or `![` @@ -281,31 +342,65 @@ private Node parseCloseBracket() { // Check to see if we have a link/image String dest = null; + String rawDest = ""; String title = null; + String rawTitle = ""; + + LinkType currentLinkType = LinkType.NULL; + char titleSymbol = Character.MIN_VALUE; + String whitespacePreDestination = ""; + String whitespacePreTitle = ""; + String whitespacePostContent = ""; // Maybe a inline link like `[foo](/uri "title")` if (scanner.next('(')) { + whitespacePreDestination = scanner.whitespaceAsString(); scanner.whitespace(); + rawDest = parseLinkDestinationRaw(scanner); dest = parseLinkDestination(scanner); + if (dest == null) { scanner.setPosition(afterClose); + whitespacePreDestination = ""; } else { - int whitespace = scanner.whitespace(); + whitespacePreTitle = scanner.whitespaceAsString(); + scanner.whitespace(); // title needs a whitespace before - if (whitespace >= 1) { + if (whitespacePreTitle.length() >= 1) { + // Capture the symbol used for the title + titleSymbol = scanner.peek(); + rawTitle = parseLinkTitleRaw(scanner); title = parseLinkTitle(scanner); - scanner.whitespace(); + + // If title isn't valid, discard the captured value of its symbol + if(title == null) { + titleSymbol = Character.MIN_VALUE; + } + whitespacePostContent = scanner.whitespaceAsString(); } + + currentLinkType = LinkType.INLINE; + if (!scanner.next(')')) { // Don't have a closing `)`, so it's not a destination and title -> reset. // Note that something like `[foo](` could be valid, `(` will just be text. scanner.setPosition(afterClose); dest = null; + rawDest = ""; title = null; + rawTitle = ""; + currentLinkType = LinkType.NULL; + titleSymbol = Character.MIN_VALUE; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; } } } + String label = null; + String rawLabel = null; + // Maybe a reference link like `[foo][bar]`, `[foo][]` or `[foo]`. // Note that even `[foo](` could be a valid link if there's a reference, which is why this is not just an `else` // here. @@ -327,13 +422,55 @@ private Node parseCloseBracket() { if (definition != null) { dest = definition.getDestination(); title = definition.getTitle(); + label = definition.getLabel(); + }else { + if(!ref.isEmpty()) { + label = ref; + } } + + currentLinkType = LinkType.REFERENCE; + + // If this reference link has a following label (like + // `[foo][bar]` or `[foo][]`, capture it for roundtrip + // purposes + Position currentPos = scanner.position(); + + scanner.setPosition(beforeClose); + scanner.next(); + + whitespacePostContent = scanner.whitespaceAsString(); + + // Check just in case there's a "[]" hanging out, but if + // there are any line breaks ignore extra symbols + // because a next pass would catch those. + if(scanner.hasNext() && scanner.peek() == '[' && !whitespacePostContent.contains("\n")) { + scanner.next(); + if(scanner.peek() == ']') { + label = ""; + } + }else { + label = null; + } + + // The raw label needs to be populated, either with the original (raw) + // link reference info or with the (literal) label information + if(label != null && !label.isEmpty() && !ref.equals(label)) { + rawLabel = ref; + }else { + rawLabel = label; + } + + // Reset scanner if no extra content was found + scanner.setPosition(currentPos); } } if (dest != null) { // If we got here, we have a link or image - Node linkOrImage = opener.image ? new Image(dest, title) : new Link(dest, title); + Node linkOrImage = opener.image ? + new Image(dest, rawDest, title, rawTitle, label, rawLabel, currentLinkType, titleSymbol, whitespacePreDestination, whitespacePreTitle, whitespacePostContent) : + new Link(dest, rawDest, title, rawTitle, label, rawLabel, currentLinkType, titleSymbol, whitespacePreDestination, whitespacePreTitle, whitespacePostContent); // Add all nodes between the opening bracket and now (closing bracket) as child nodes of the link Node node = opener.node.getNext(); @@ -348,7 +485,7 @@ private Node parseCloseBracket() { } // Process delimiters such as emphasis inside link/image - processDelimiters(opener.previousDelimiter); + processDelimiters(opener.previousDelimiter, prefix); mergeChildTextNodes(linkOrImage); // We don't need the corresponding text node anymore, we turned it into a link/image node opener.node.unlink(); @@ -373,7 +510,7 @@ private Node parseCloseBracket() { removeLastBracket(); scanner.setPosition(afterClose); - return text(scanner.getSource(beforeClose, afterClose)); + return text(scanner.getSource(beforeClose, afterClose), prefix, ""); } } @@ -409,6 +546,18 @@ private String parseLinkDestination(Scanner scanner) { return Escaping.unescapeString(dest); } + + private String parseLinkDestinationRaw(Scanner rawScanner) { + Position start = rawScanner.position(); + if(!LinkScanner.scanLinkDestination(rawScanner)) { + return null; + } + + String dest = rawScanner.getSource(start, scanner.position()).getContent(); + rawScanner.setPosition(start); + + return dest; + } /** * Attempt to parse link title (sans quotes), returning the string or null if no match. @@ -424,6 +573,22 @@ private String parseLinkTitle(Scanner scanner) { String title = rawTitle.substring(1, rawTitle.length() - 1); return Escaping.unescapeString(title); } + + /** + * Attempt to parse the link title in raw form, returning the raw string or null if no match. + */ + private String parseLinkTitleRaw(Scanner rawScanner) { + Position start = rawScanner.position(); + if(!LinkScanner.scanLinkTitle(rawScanner)) { + return null; + } + + String rawTitle = scanner.getSource(start, scanner.position()).getContent(); + + rawScanner.setPosition(start); + + return rawTitle; + } /** * Attempt to parse a link label, returning the label between the brackets or null. @@ -466,32 +631,69 @@ private Node parseLineBreak() { * Parse the next character as plain text, and possibly more if the following characters are non-special. */ private Node parseText() { + return parseText(""); + } + + private Node parseText(String prefix) { + String preContentWhitespace = ""; + + if(!prefix.isEmpty()) { + // Capture whitespace and anything that might be a block quote for roundtrip purposes + if(prefix.isBlank() || prefix.contains(">")) { + preContentWhitespace = prefix; + } + } + + StringBuilder postContentWhitespace = new StringBuilder(); + Position start = scanner.position(); scanner.next(); + char c; while (true) { c = scanner.peek(); - if (c == Scanner.END || specialCharacters.get(c)) { + + if(c == Scanner.END) { + break; + } + + if(c != ' ' && c != '\t' && c != '\n' && postContentWhitespace.length() > 0) { + postContentWhitespace.setLength(0); + } + + if (specialCharacters.get(c)) { break; } + + if(c == ' ' || c == '\t') { + postContentWhitespace.append(c); + } + scanner.next(); } - - SourceLines source = scanner.getSource(start, scanner.position()); + + SourceLines source = scanner.getSource(start, scanner.position(), prefix.length()); String content = source.getContent(); if (c == '\n') { // We parsed until the end of the line. Trim any trailing spaces and remember them (for hard line breaks). int end = Parsing.skipBackwards(' ', content, content.length() - 1, 0) + 1; trailingSpaces = content.length() - end; + if(postContentWhitespace.length() == 0) { + postContentWhitespace.append(content.subSequence(end, content.length())); + } content = content.substring(0, end); } else if (c == Scanner.END) { // For the last line, both tabs and spaces are trimmed for some reason (checked with commonmark.js). int end = Parsing.skipSpaceTabBackwards(content, content.length() - 1, 0) + 1; + if(postContentWhitespace.length() == 0) { + postContentWhitespace.append(content.subSequence(end, content.length())); + } content = content.substring(0, end); } - - Text text = new Text(content); + + Text text = new Text(content, content, preContentWhitespace, postContentWhitespace.toString()); + text.setSourceSpans(source.getSourceSpans()); return text; } @@ -502,7 +704,7 @@ private Node parseText() { * * @return information about delimiter run, or {@code null} */ - private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar) { + private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar, String prefix) { int before = scanner.peekPreviousCodePoint(); Position start = scanner.position(); @@ -517,8 +719,16 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char List delimiters = new ArrayList<>(); scanner.setPosition(start); Position positionBefore = start; + while (scanner.next(delimiterChar)) { - delimiters.add(text(scanner.getSource(positionBefore, scanner.position()))); + Text delimText = text(scanner.getSource(positionBefore, scanner.position()), prefix, ""); + + // Erase prefix after first line + if(prefix.length() > 0) { + prefix = ""; + } + + delimiters.add(delimText); positionBefore = scanner.position(); } @@ -548,6 +758,10 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char } private void processDelimiters(Delimiter stackBottom) { + processDelimiters(stackBottom, ""); + } + + private void processDelimiters(Delimiter stackBottom, String prefix) { Map openersBottom = new HashMap<>(); @@ -576,7 +790,13 @@ private void processDelimiters(Delimiter stackBottom) { while (opener != null && opener != stackBottom && opener != openersBottom.get(delimiterChar)) { if (opener.canOpen() && opener.delimiterChar == openingDelimiterChar) { potentialOpenerFound = true; - usedDelims = delimiterProcessor.process(opener, closer); + usedDelims = delimiterProcessor.process(opener, closer, prefix); + + String openerWhitespace = opener.characters.get(0).whitespacePreContent(); + if(!openerWhitespace.isBlank()) { + closer.characters.get(0).setWhitespace(openerWhitespace, ""); + } + if (usedDelims > 0) { openerFound = true; break; @@ -711,9 +931,11 @@ private void mergeTextNodesInclusive(Node fromNode, Node toNode) { } private void mergeIfNeeded(Text first, Text last, int textLength) { - if (first != null && last != null && first != last) { + if (first != null && last != null && first != last) { StringBuilder sb = new StringBuilder(textLength); + StringBuilder sb2 = new StringBuilder(textLength); sb.append(first.getLiteral()); + sb2.append(first.getRaw()); SourceSpans sourceSpans = null; if (includeSourceSpans) { sourceSpans = new SourceSpans(); @@ -723,6 +945,7 @@ private void mergeIfNeeded(Text first, Text last, int textLength) { Node stop = last.getNext(); while (node != stop) { sb.append(((Text) node).getLiteral()); + sb2.append(((Text)node).getRaw()); if (sourceSpans != null) { sourceSpans.addAll(node.getSourceSpans()); } @@ -732,7 +955,9 @@ private void mergeIfNeeded(Text first, Text last, int textLength) { unlink.unlink(); } String literal = sb.toString(); + String raw = sb2.toString(); first.setLiteral(literal); + first.setRaw(raw); if (sourceSpans != null) { first.setSourceSpans(sourceSpans.getSourceSpans()); } diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java index 2bceb7549..96790a797 100644 --- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java @@ -1,5 +1,8 @@ package org.commonmark.internal; +import java.util.ArrayList; +import java.util.List; + import org.commonmark.internal.inline.Position; import org.commonmark.internal.inline.Scanner; import org.commonmark.internal.util.Escaping; @@ -9,9 +12,6 @@ import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import java.util.ArrayList; -import java.util.List; - /** * Parser for link reference definitions at the beginning of a paragraph. * @@ -26,20 +26,37 @@ public class LinkReferenceDefinitionParser { private final List sourceSpans = new ArrayList<>(); private StringBuilder label; + private String extendedLabel = null; private String destination; + private String rawDestination; private char titleDelimiter; private StringBuilder title; private boolean referenceValid = false; + private List rawParagraphLines = new ArrayList<>(); + + private int previousLineIndex = 0; + private String whitespacePreLabel = ""; + private String whitespacePreDestination = ""; + private String whitespacePreTitle = ""; + private String whitespacePostTitle = ""; public void parse(SourceLine line) { - paragraphLines.add(line); - if (state == State.PARAGRAPH) { + paragraphLines.add(line.getLiteralLine()); + rawParagraphLines.add(line); + + if (state == State.PARAGRAPH) { // We're in a paragraph now. Link reference definitions can only appear at the beginning, so once // we're in a paragraph, there's no going back. return; } Scanner scanner = Scanner.of(SourceLines.of(line)); + if(line.getSourceSpan() != null) { + // Ensure that LinkReferenceDefinition objects can have populated source spans + // Source spans track literal lines, not raw lines + addSourceSpan(line.getLiteralLine().getSourceSpan()); + } + while (scanner.hasNext()) { boolean success; switch (state) { @@ -48,14 +65,35 @@ public void parse(SourceLine line) { break; } case LABEL: { + // Capture line number to determine if line breaks occur + if(line.getSourceSpan() != null) { + previousLineIndex = line.getSourceSpan().getLineIndex(); + } + success = label(scanner); break; } case DESTINATION: { + // Check for any line breaks + if(line.getSourceSpan() != null) { + if(line.getSourceSpan().getLineIndex() > previousLineIndex) { + whitespacePreDestination = whitespacePreDestination + "\n"; + previousLineIndex++; + } + } + success = destination(scanner); break; } case START_TITLE: { + // Check for any line breaks + if(line.getSourceSpan() != null) { + if(line.getSourceSpan().getLineIndex() > previousLineIndex) { + whitespacePreTitle = whitespacePreTitle + "\n"; + previousLineIndex++; + } + } + success = startTitle(scanner); break; } @@ -85,10 +123,22 @@ public void addSourceSpan(SourceSpan sourceSpan) { SourceLines getParagraphLines() { return SourceLines.of(paragraphLines); } + + SourceLines getRawParagraphLines() { + return SourceLines.of(rawParagraphLines); + } + + void setRawParagraphLines(SourceLines rawParagraphLines) { + this.rawParagraphLines = rawParagraphLines.getLines(); + } List getParagraphSourceSpans() { return sourceSpans; } + + String getExtendedLabel() { + return extendedLabel; + } List getDefinitions() { finishReference(); @@ -100,8 +150,16 @@ State getState() { } private boolean startDefinition(Scanner scanner) { - scanner.whitespace(); - if (!scanner.next('[')) { + // Capture whitespace for roundtrip purposes + whitespacePreLabel = scanner.whitespaceAsString(); + + String whitespaceCheck = scanner.alignToLiteral(); + + if(!whitespaceCheck.isEmpty()) { + whitespacePreLabel = whitespaceCheck; + } + + if (!scanner.next('[')) { return false; } @@ -144,7 +202,7 @@ private boolean label(Scanner scanner) { state = State.DESTINATION; - scanner.whitespace(); + whitespacePreDestination = scanner.whitespaceAsString(); return true; } else { return false; @@ -152,24 +210,26 @@ private boolean label(Scanner scanner) { } private boolean destination(Scanner scanner) { - scanner.whitespace(); + whitespacePreDestination = whitespacePreDestination + + scanner.whitespaceAsString(); Position start = scanner.position(); if (!LinkScanner.scanLinkDestination(scanner)) { return false; } - String rawDestination = scanner.getSource(start, scanner.position()).getContent(); + rawDestination = scanner.getSource(start, scanner.position()).getContent(); destination = rawDestination.startsWith("<") ? rawDestination.substring(1, rawDestination.length() - 1) : rawDestination; - int whitespace = scanner.whitespace(); + whitespacePreTitle = scanner.whitespaceAsString(); if (!scanner.hasNext()) { // Destination was at end of line, so this is a valid reference for sure (and maybe a title). // If not at end of line, wait for title to be valid first. referenceValid = true; paragraphLines.clear(); - } else if (whitespace == 0) { + rawParagraphLines.clear(); + } else if (whitespacePreTitle.length() == 0) { // spec: The title must be separated from the link destination by whitespace return false; } @@ -179,8 +239,8 @@ private boolean destination(Scanner scanner) { } private boolean startTitle(Scanner scanner) { - scanner.whitespace(); - if (!scanner.hasNext()) { + whitespacePreTitle = whitespacePreTitle + scanner.whitespaceAsString(); + if (!scanner.hasNext()) { state = State.START_DEFINITION; return true; } @@ -229,14 +289,17 @@ private boolean title(Scanner scanner) { // Skip delimiter character scanner.next(); - scanner.whitespace(); + whitespacePostTitle = scanner.whitespaceAsString(); if (scanner.hasNext()) { + title = null; + whitespacePostTitle = ""; // spec: No further non-whitespace characters may occur on the line. return false; } referenceValid = true; finishReference(); paragraphLines.clear(); + rawParagraphLines.clear(); // See if there's another definition. state = State.START_DEFINITION; @@ -248,9 +311,14 @@ private void finishReference() { return; } + if(rawDestination.isEmpty()) { + rawDestination = destination; + } + String d = Escaping.unescapeString(destination); String t = title != null ? Escaping.unescapeString(title.toString()) : null; - LinkReferenceDefinition definition = new LinkReferenceDefinition(label.toString(), d, t); + String rawTitle = title != null ? title.toString() : ""; + LinkReferenceDefinition definition = new LinkReferenceDefinition(label.toString(), d, rawDestination, t, rawTitle, titleDelimiter, whitespacePreLabel, whitespacePreDestination, whitespacePreTitle, whitespacePostTitle); definition.setSourceSpans(sourceSpans); sourceSpans.clear(); definitions.add(definition); @@ -259,6 +327,8 @@ private void finishReference() { referenceValid = false; destination = null; title = null; + whitespacePreDestination = ""; + whitespacePostTitle = ""; } enum State { diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java index f6702518b..f2b438b4e 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java @@ -1,8 +1,18 @@ package org.commonmark.internal; import org.commonmark.internal.util.Parsing; -import org.commonmark.node.*; -import org.commonmark.parser.block.*; +import org.commonmark.node.Block; +import org.commonmark.node.BulletList; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.OrderedList; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockParser; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class ListBlockParser extends AbstractBlockParser { @@ -11,6 +21,17 @@ public class ListBlockParser extends AbstractBlockParser { private boolean hadBlankLine; private int linesAfterBlank; + private static String currentRawNumber; + + /* + * When gathering data for roundtrip parsing, there is no easy way to know whether + * a blank list item is a precursor to more content (CommonMark test case 248) or + * if it stands alone (CommonMark test case 252). + * Luckily, if the first line is blank, any further content which doesn't start a + * new list item indicates that the first line is split, not standalone. + */ + private static boolean firstLineBlank = false; + public ListBlockParser(ListBlock block) { this.block = block; } @@ -114,9 +135,28 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { case '-': case '+': case '*': - if (isSpaceTabOrEnd(line, index + 1)) { + if (isSpaceTabOrEnd(line, index + 1)) { + // Collect any pre-content whitespace in the first line for + // roundtrip purposes. + String preContentWhitespace = Parsing.collectWhitespace(line, index + 1, line.length()); + + // AST: Lists can be separated from their delimiter by a blank line, make + // sure to capture this for roundtrip purposes if it occurs + if(line.subSequence(index + 1, line.length()).toString().isBlank()) { + firstLineBlank = true; + }else { + firstLineBlank = false; + } + + String preBlockWhitespace = line.subSequence(0, index).toString(); + + if(!preBlockWhitespace.isBlank()) { + preBlockWhitespace = ""; + } + BulletList bulletList = new BulletList(); bulletList.setBulletMarker(c); + bulletList.setWhitespace(preBlockWhitespace, preContentWhitespace); return new ListMarkerData(bulletList, index + 1); } else { return null; @@ -153,9 +193,31 @@ private static ListMarkerData parseOrderedList(CharSequence line, int index) { case ')': if (digits >= 1 && isSpaceTabOrEnd(line, i + 1)) { String number = line.subSequence(index, i).toString(); + + // AST: Lists can be separated from their delimiter by a blank line, make + // sure to capture this for roundtrip purposes if it occurs + if(line.subSequence(index + 1, line.length()).toString().isBlank()) { + firstLineBlank = true; + }else { + firstLineBlank = false; + } + + // Collect any pre-content whitespace in the first line for + // roundtrip purposes. + String preContentWhitespace = Parsing.collectWhitespace(line, i+1, line.length()); + String preBlockWhitespace = line.subSequence(0, index).toString(); + + if(!preBlockWhitespace.isBlank()) { + preBlockWhitespace = ""; + } + OrderedList orderedList = new OrderedList(); orderedList.setStartNumber(Integer.parseInt(number)); + // AST: Actual list number can be anything, it is + // only auto-incremented during rendering + currentRawNumber = number; orderedList.setDelimiter(c); + orderedList.setWhitespace(preBlockWhitespace, preContentWhitespace); return new ListMarkerData(orderedList, i + 1); } else { return null; @@ -217,8 +279,12 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar } int newColumn = listData.contentColumn; - ListItemParser listItemParser = new ListItemParser(newColumn - state.getColumn()); - + String preBlockWhitespace = listData.listBlock.whitespacePreBlock(); + String preContentWhitespace = listData.listBlock.whitespacePreContent(); + + ListItemParser listItemParser = new ListItemParser(newColumn - state.getColumn(), currentRawNumber, firstLineBlank, preBlockWhitespace, preContentWhitespace); + firstLineBlank = false; + // prepend the list block if needed if (!(matched instanceof ListBlockParser) || !(listsMatch((ListBlock) matched.getBlock(), listData.listBlock))) { diff --git a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java index 6f03770b3..45a80c71f 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java @@ -1,5 +1,6 @@ package org.commonmark.internal; +import org.commonmark.node.BlankLine; import org.commonmark.node.Block; import org.commonmark.node.ListBlock; import org.commonmark.node.ListItem; @@ -20,8 +21,19 @@ public class ListItemParser extends AbstractBlockParser { private boolean hadBlankLine; + // Helps distinguish multi-line line items, as described in 
ListBlockParser
+ private boolean firstLineBlank = false; + public ListItemParser(int contentIndent) { this.contentIndent = contentIndent; + block.setRawNumber(""); + } + + public ListItemParser(int contentIndent, String currentRawNumber, boolean firstLineBlank, String whitespacePreBlock, String whitespacePreContent) { + this(contentIndent); + block.setRawNumber(currentRawNumber); + this.firstLineBlank = firstLineBlank; + block.setWhitespace(whitespacePreBlock, whitespacePreContent); } @Override @@ -52,18 +64,24 @@ public Block getBlock() { @Override public BlockContinue tryContinue(ParserState state) { if (state.isBlank()) { - if (block.getFirstChild() == null) { + if (block.getFirstChild() == null || block.getFirstChild() instanceof BlankLine) { // Blank line after empty list item return BlockContinue.none(); } else { Block activeBlock = state.getActiveBlockParser().getBlock(); // If the active block is a code block, blank lines in it should not affect if the list is tight. hadBlankLine = activeBlock instanceof Paragraph || activeBlock instanceof ListItem; + return BlockContinue.atIndex(state.getNextNonSpaceIndex()); } } if (state.getIndent() >= contentIndent) { + if(firstLineBlank) { + String whitespacePreContent = block.whitespacePreContent() + "\n"; + block.setWhitespace(block.whitespacePreBlock(), whitespacePreContent); + firstLineBlank = false; + } return BlockContinue.atColumn(state.getColumn() + contentIndent); } else { // Note: We'll hit this case for lazy continuation lines, they will get added later. diff --git a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java index 89328ef2a..e3bb33a61 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java @@ -1,5 +1,8 @@ package org.commonmark.internal; +import java.util.List; + +import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.LinkReferenceDefinition; import org.commonmark.node.Paragraph; @@ -11,13 +14,25 @@ import org.commonmark.parser.block.BlockContinue; import org.commonmark.parser.block.ParserState; -import java.util.List; - public class ParagraphParser extends AbstractBlockParser { private final Paragraph block = new Paragraph(); private final LinkReferenceDefinitionParser linkReferenceDefinitionParser = new LinkReferenceDefinitionParser(); + private String postContentWhitespace = ""; + private String postBlockWhitespace = ""; + + // Preserve original default constructor by explicitly defining one + public ParagraphParser() { + super(); + } + + public ParagraphParser(String preBlockWhitespace) { + block.setWhitespace(preBlockWhitespace); + postContentWhitespace = ""; + postBlockWhitespace = ""; + } + @Override public boolean canHaveLazyContinuationLines() { return true; @@ -30,7 +45,19 @@ public Block getBlock() { @Override public BlockContinue tryContinue(ParserState state) { + char setextCheck = Character.MIN_VALUE; + postBlockWhitespace = block.whitespacePostBlock(); + if (!state.isBlank()) { + + if(state.getLine() != null && state.getLine().getContent().toString().stripLeading().length() != 0) { + setextCheck = state.getLine().getContent().toString().stripLeading().charAt(0); + } + + if(setextCheck != Character.MIN_VALUE && setextCheck != '=' && setextCheck != '-') { + // Collect post-content whitespace for roundtrip purposes + postContentWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), state.getLine().getContent().length() - 1, 0); + } return BlockContinue.atIndex(state.getIndex()); } else { return BlockContinue.none(); @@ -51,6 +78,8 @@ public void addSourceSpan(SourceSpan sourceSpan) { @Override public void closeBlock() { + block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), postContentWhitespace, postBlockWhitespace); + if (linkReferenceDefinitionParser.getParagraphLines().isEmpty()) { block.unlink(); } else { @@ -62,13 +91,21 @@ public void closeBlock() { public void parseInlines(InlineParser inlineParser) { SourceLines lines = linkReferenceDefinitionParser.getParagraphLines(); if (!lines.isEmpty()) { - inlineParser.parse(lines, block); + inlineParser.parse(getRawParagraphLines(), block); } } public SourceLines getParagraphLines() { return linkReferenceDefinitionParser.getParagraphLines(); } + + public SourceLines getRawParagraphLines() { + return linkReferenceDefinitionParser.getRawParagraphLines(); + } + + public void setRawParagraphLines(SourceLines rawParagraphLines) { + linkReferenceDefinitionParser.setRawParagraphLines(rawParagraphLines); + } public List getDefinitions() { return linkReferenceDefinitionParser.getDefinitions(); diff --git a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java index 0fe8065bb..13f9b70b8 100644 --- a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java @@ -73,4 +73,10 @@ private DelimiterProcessor findProcessor(int len) { public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return findProcessor(openingRun.length()).process(openingRun, closingRun); } + + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java index 88ad41ac9..78192e196 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java @@ -1,13 +1,33 @@ package org.commonmark.internal; +import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.ThematicBreak; -import org.commonmark.parser.block.*; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.AbstractBlockParserFactory; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.BlockStart; +import org.commonmark.parser.block.MatchedBlockParser; +import org.commonmark.parser.block.ParserState; public class ThematicBreakParser extends AbstractBlockParser { private final ThematicBreak block = new ThematicBreak(); + // Preserve original default constructor by explicitly defining one + public ThematicBreakParser() { + super(); + } + + public ThematicBreakParser(CharSequence content) { + block.setContent(content); + } + + public ThematicBreakParser(CharSequence content, String... whitespace) { + block.setContent(content); + block.setWhitespace(whitespace); + } + @Override public Block getBlock() { return block; @@ -28,8 +48,13 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar } int nextNonSpace = state.getNextNonSpaceIndex(); CharSequence line = state.getLine().getContent(); + + // Collect pre- and post-content whitespace for roundtrip purposes + String preContentWhitespace = Parsing.collectWhitespace(line, 0, nextNonSpace); + String postContentWhitespace = Parsing.collectWhitespaceBackwards(line, line.length() - 1, nextNonSpace); + if (isThematicBreak(line, nextNonSpace)) { - return BlockStart.of(new ThematicBreakParser()).atIndex(line.length()); + return BlockStart.of(new ThematicBreakParser(line, "", preContentWhitespace, postContentWhitespace)).atIndex(line.length()); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java index ecfd2d972..ee38b5437 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java @@ -1,11 +1,12 @@ package org.commonmark.internal.inline; +import java.util.regex.Pattern; + import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; import org.commonmark.node.Text; import org.commonmark.parser.SourceLines; -import java.util.regex.Pattern; - /** * Attempt to parse an autolink (URL or email in pointy brackets). */ @@ -39,9 +40,16 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { Text text = new Text(content); text.setSourceSpans(textSource.getSourceSpans()); link.appendChild(text); + link.setLinkType(LinkType.AUTOLINK); return ParsedInline.of(link, scanner.position()); } } return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Autolinks do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java index f57a67a74..8e6d83853 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java @@ -24,12 +24,18 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { char next = scanner.peek(); if (next == '\n') { scanner.next(); - return ParsedInline.of(new HardLineBreak(), scanner.position()); + return ParsedInline.of(new HardLineBreak(true), scanner.position()); } else if (ESCAPABLE.matcher(String.valueOf(next)).matches()) { scanner.next(); - return ParsedInline.of(new Text(String.valueOf(next)), scanner.position()); + return ParsedInline.of(new Text(String.valueOf(next), "\\" + String.valueOf(next), "", ""), scanner.position()); } else { return ParsedInline.of(new Text("\\"), scanner.position()); } } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Backslash-escaped characters do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java index ad079444a..64a1a178c 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java @@ -14,6 +14,13 @@ public class BackticksInlineParser implements InlineContentParser { public ParsedInline tryParse(InlineParserState inlineParserState) { Scanner scanner = inlineParserState.scanner(); Position start = scanner.position(); + String preBlockWhitespace = ""; + + char listCheck = scanner.peek(); + if(listCheck == '-' || listCheck == '*' || listCheck == '+') { + preBlockWhitespace = listCheck + scanner.whitespaceAsString(); + } + int openingTicks = scanner.matchMultiple('`'); Position afterOpening = scanner.position(); @@ -23,7 +30,11 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { if (count == openingTicks) { Code node = new Code(); +node.setNumBackticks(count); + String content = scanner.getSource(afterOpening, beforeClosing).getContent(); + // Capture raw string for roundtrip rendering + node.setRaw(content); content = content.replace('\n', ' '); // spec: If the resulting string both begins and ends with a space character, but does not consist @@ -42,7 +53,13 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { // If we got here, we didn't find a matching closing backtick sequence. SourceLines source = scanner.getSource(start, afterOpening); - Text text = new Text(source.getContent()); + Text text = new Text(source.getContent(), source.getContent(), preBlockWhitespace, ""); return ParsedInline.of(text, afterOpening); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Backticks do not have significant prefix values + return tryParse(inlineParserState); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java new file mode 100644 index 000000000..8b59bf574 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java @@ -0,0 +1,28 @@ +package org.commonmark.internal.inline; + +import org.commonmark.node.BlankLine; +import org.commonmark.node.Block; +import org.commonmark.parser.block.AbstractBlockParser; +import org.commonmark.parser.block.BlockContinue; +import org.commonmark.parser.block.ParserState; + +public class BlankLineParser extends AbstractBlockParser { + + private BlankLine block; + + public BlankLineParser(String raw) { + block = new BlankLine(raw); + } + + @Override + public Block getBlock() { + return block; + } + + @Override + public BlockContinue tryContinue(ParserState parserState) { + // Blank lines do not become active parsers, so they cannot be continued + return null; + } + +} \ No newline at end of file diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java index 493e4299c..1a32b1f5e 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java @@ -63,4 +63,49 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return usedDelimiters; } + + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + // "multiple of 3" rule for internal delimiter runs + if ((openingRun.canClose() || closingRun.canOpen()) && + closingRun.originalLength() % 3 != 0 && + (openingRun.originalLength() + closingRun.originalLength()) % 3 == 0) { + return 0; + } + + int usedDelimiters; + Node emphasis; + + // Pre-content whitespace in emphasis is primarily used during thematic breaks + String preContentWhitespace = prefix; + + if(!openingRun.getOpener().whitespacePreContent().isEmpty() && + openingRun.getOpener().whitespacePreContent().isBlank()) { + preContentWhitespace = openingRun.getOpener().whitespacePreContent(); + } + + // calculate actual number of delimiters used from this closer + if (openingRun.length() >= 2 && closingRun.length() >= 2) { + usedDelimiters = 2; + emphasis = new StrongEmphasis(String.valueOf(delimiterChar) + delimiterChar, preContentWhitespace); + } else { + usedDelimiters = 1; + emphasis = new Emphasis(String.valueOf(delimiterChar), preContentWhitespace); + } + + SourceSpans sourceSpans = SourceSpans.empty(); + sourceSpans.addAllFrom(openingRun.getOpeners(usedDelimiters)); + + Text opener = openingRun.getOpener(); + for (Node node : Nodes.between(opener, closingRun.getCloser())) { + emphasis.appendChild(node); + sourceSpans.addAll(node.getSourceSpans()); + } + + sourceSpans.addAllFrom(closingRun.getClosers(usedDelimiters)); + + emphasis.setSourceSpans(sourceSpans.getSourceSpans()); + opener.insertAfter(emphasis); + + return usedDelimiters; + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java index c29b8694f..b47b87546 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java @@ -45,9 +45,15 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Inline entities do not have significant prefix values + return tryParse(inlineParserState); + } private ParsedInline entity(Scanner scanner, Position start) { String text = scanner.getSource(start, scanner.position()).getContent(); - return ParsedInline.of(new Text(Html5Entities.entityToString(text)), scanner.position()); + return ParsedInline.of(new Text(Html5Entities.entityToString(text), text, "", ""), scanner.position()); } } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java index 605901c22..14a773017 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java @@ -65,6 +65,12 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { return ParsedInline.none(); } + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState, String prefix) { + // Inline HTML does not have significant prefix values + return tryParse(inlineParserState); + } private static ParsedInline htmlInline(Position start, Scanner scanner) { String text = scanner.getSource(start, scanner.position()).getContent(); diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java index 755ee3135..9fba0c949 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java @@ -3,4 +3,6 @@ public interface InlineContentParser { ParsedInline tryParse(InlineParserState inlineParserState); + + ParsedInline tryParse(InlineParserState inlineParserState, String prefix); } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java index 9de96a587..730de8682 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java @@ -1,12 +1,12 @@ package org.commonmark.internal.inline; +import java.util.List; + import org.commonmark.internal.util.CharMatcher; import org.commonmark.node.SourceSpan; import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; -import java.util.List; - public class Scanner { /** @@ -193,6 +193,30 @@ public int whitespace() { } } + /** + * Capture next series of whitespace + * @return Consecutive whitespace as String, or empty String if no whitespace found + */ + public String whitespaceAsString() { + StringBuilder sb = new StringBuilder(0); + while (true) { + char c = peek(); + switch (c) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + sb.append(c); + next(); + break; + default: + return sb.toString(); + } + } + } + public int find(char c) { int count = 0; while (true) { @@ -237,6 +261,10 @@ public void setPosition(Position position) { // For cases where the caller appends the result to a StringBuilder, we could offer another method to avoid some // unnecessary copying. public SourceLines getSource(Position begin, Position end) { + return getSource(begin, end, 0); + } + + public SourceLines getSource(Position begin, Position end, int offset) { if (begin.lineIndex == end.lineIndex) { // Shortcut for common case of text from a single line SourceLine line = lines.get(begin.lineIndex); @@ -244,7 +272,14 @@ public SourceLines getSource(Position begin, Position end) { SourceSpan newSourceSpan = null; SourceSpan sourceSpan = line.getSourceSpan(); if (sourceSpan != null) { - newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), sourceSpan.getColumnIndex() + begin.index, newContent.length()); + int newColumnIndex = sourceSpan.getColumnIndex() + begin.index; + + // Make sure SourceSpan is still tracking literal line, not its prefix + if(offset > 0) { + newColumnIndex -= offset; + } + + newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), newColumnIndex, newContent.length()); } return SourceLines.of(SourceLine.of(newContent, newSourceSpan)); } else { @@ -263,6 +298,18 @@ public SourceLines getSource(Position begin, Position end) { return sourceLines; } } + + // Many lines are passed around as raw strings, with an index value indicating + // where the literal string begins. Calling this method returns only the literal + // portion of the line, or an empty string if that is not possible. + public String alignToLiteral() { + if(index >= line.getLiteralIndex() || line.getLiteralIndex() == 0) { + return ""; + }else { + index = line.getLiteralIndex(); + return line.substring(0, index).getContent().toString(); + } + } private void setLine(SourceLine line) { this.line = line; diff --git a/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java b/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java index e02ecea7c..3bebf037d 100644 --- a/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java +++ b/commonmark/src/main/java/org/commonmark/internal/renderer/text/OrderedListHolder.java @@ -5,11 +5,13 @@ public class OrderedListHolder extends ListHolder { private final char delimiter; private int counter; + private String rawNumber; public OrderedListHolder(ListHolder parent, OrderedList list) { super(parent); delimiter = list.getDelimiter(); counter = list.getStartNumber(); + rawNumber = list.getRawNumber(); } public char getDelimiter() { @@ -19,6 +21,10 @@ public char getDelimiter() { public int getCounter() { return counter; } + + public String getRawNumber() { + return rawNumber; + } public void increaseCounter() { counter++; diff --git a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java index 3ca34c5f0..f492e9f1c 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java @@ -70,7 +70,9 @@ public static boolean scanLinkTitle(Scanner scanner) { } char endDelimiter; - switch (scanner.peek()) { + char c = scanner.peek(); + + switch (c) { case '"': endDelimiter = '"'; break; @@ -83,6 +85,10 @@ public static boolean scanLinkTitle(Scanner scanner) { default: return false; } + +// if(c != '(' && !(scanner.hasNext() && scanner.peek() == '(')) { +// scanner.next(); +// } scanner.next(); if (!scanLinkTitleContent(scanner, endDelimiter)) { @@ -158,6 +164,11 @@ private static boolean scanLinkDestinationWithBalancedParens(Scanner scanner) { } empty = false; } + + if(parens > 0) { + return false; + } + return true; } } diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java index 8b02e99b1..6d959c233 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java @@ -226,6 +226,56 @@ public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int last return lastIndex - 1; } + public static String collectWhitespace(CharSequence s, int startIndex, int lastIndex) { + StringBuilder whitespace = new StringBuilder(lastIndex - startIndex); + + for (int i = startIndex; i < lastIndex; i++) { + switch (s.charAt(i)) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + whitespace.append(s.charAt(i)); + break; + default: + return whitespace.toString(); + } + } + return whitespace.toString(); + } + + public static String collectWhitespaceBackwards(CharSequence s, int startIndex, int lastIndex) { + StringBuilder whitespace = new StringBuilder(startIndex - lastIndex); + + for (int i = startIndex; i >= lastIndex; i--) { + switch (s.charAt(i)) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + whitespace.append(s.charAt(i)); + break; + default: + return whitespace.toString(); + } + } + return whitespace.toString(); + } + + public static String generateSpaces(int numberOfSpaces) { + StringBuilder spaces = new StringBuilder(numberOfSpaces); + + for(int i = 0; i < numberOfSpaces; i++) { + spaces.append(" "); + } + + return spaces.toString(); + } + private static int findNonSpace(CharSequence s, int startIndex) { int length = s.length(); for (int i = startIndex; i < length; i++) { diff --git a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java index 7edd635d7..941a6d45e 100644 --- a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java +++ b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java @@ -8,6 +8,11 @@ */ public abstract class AbstractVisitor implements Visitor { + @Override + public void visit(BlankLine blankLine) { + visitChildren(blankLine); + } + @Override public void visit(BlockQuote blockQuote) { visitChildren(blockQuote); diff --git a/commonmark/src/main/java/org/commonmark/node/BlankLine.java b/commonmark/src/main/java/org/commonmark/node/BlankLine.java new file mode 100644 index 000000000..4f63c1686 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/node/BlankLine.java @@ -0,0 +1,52 @@ +package org.commonmark.node; + +public class BlankLine extends Block { + + private String raw; + + public BlankLine(String rawContent) { + raw = rawContent; + } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + @Override + public String whitespacePreBlock() { + // Blank lines contain any whitespace as part of their raw content + return ""; + } + + @Override + public String whitespacePreContent() { + // Blank lines contain any whitespace as part of their raw content + return ""; + } + + @Override + public String whitespacePostContent() { + // Blank lines contain any whitespace as part of their raw content + return ""; + } + + @Override + public String whitespacePostBlock() { + // Blank lines contain any whitespace as part of their raw content + return ""; + } + + @Override + public void setWhitespace(String... newWhitespace) { + // Blank lines contain any whitespace as part of their raw content + } + + @Override + public void accept(Visitor visitor) { + visitor.visit(this); + } +} diff --git a/commonmark/src/main/java/org/commonmark/node/Block.java b/commonmark/src/main/java/org/commonmark/node/Block.java index 753447c5c..976245d96 100644 --- a/commonmark/src/main/java/org/commonmark/node/Block.java +++ b/commonmark/src/main/java/org/commonmark/node/Block.java @@ -16,4 +16,40 @@ protected void setParent(Node parent) { } super.setParent(parent); } + + // Roundtrip rendering requires capturing various pieces before and after blocks and their contents + public abstract String whitespacePreBlock(); + public abstract String whitespacePreContent(); + public abstract String whitespacePostContent(); + public abstract String whitespacePostBlock(); + public abstract void setWhitespace(String...newWhitespace); + + /** + * All block elements can be impacted structurally by whitespace in some way or another. + * For convenience, this method allows passing any number of strings. However, these strings + * are interpreted as linearly replacing one of 4 pre-defined positions (i.e + * 2 values replace [0] and [1], 3 replace [0], [1], and [2] etc.). These 4 + * values correspond to: + * 0. Pre-block element whitespace + * 1. Pre-content whitespace + * 2. Post-content whitespace + * 3. Post-block element whitespace + * + * If a caller passes more than 4 values, all values past the fourth will be ignored. + * @param newWhitespace + * @return Prepared array of structural whitespace + */ + protected String[] prepareStructuralWhitespace(String... newWhitespace) { + String[] container = {"", "", "", ""}; + + if(newWhitespace.length == 4) { + container = newWhitespace; + }else { + for(int i = 0; i < newWhitespace.length; i++) { + container[i] = newWhitespace[i]; + } + } + + return container; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java index 160f25ae2..bed8119c0 100644 --- a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java +++ b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java @@ -1,9 +1,39 @@ package org.commonmark.node; public class BlockQuote extends Block { - + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; + @Override public void accept(Visitor visitor) { visitor.visit(this); } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/BulletList.java b/commonmark/src/main/java/org/commonmark/node/BulletList.java index 127862312..816a61096 100644 --- a/commonmark/src/main/java/org/commonmark/node/BulletList.java +++ b/commonmark/src/main/java/org/commonmark/node/BulletList.java @@ -1,6 +1,12 @@ package org.commonmark.node; public class BulletList extends ListBlock { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; private char bulletMarker; @@ -17,4 +23,27 @@ public void setBulletMarker(char bulletMarker) { this.bulletMarker = bulletMarker; } + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Code.java b/commonmark/src/main/java/org/commonmark/node/Code.java index 0b47ecb71..268667978 100644 --- a/commonmark/src/main/java/org/commonmark/node/Code.java +++ b/commonmark/src/main/java/org/commonmark/node/Code.java @@ -3,6 +3,9 @@ public class Code extends Node { private String literal; + private String raw; + private boolean strippedSpaces; + private int numBackticks; public Code() { } @@ -23,4 +26,28 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public boolean hasStrippedSpaces() { + return strippedSpaces; + } + + public void setStrippedSpaces(boolean strippedSpaces) { + this.strippedSpaces = strippedSpaces; + } + + public int getNumBackticks() { + return numBackticks; + } + + public void setNumBackticks(int numBackticks) { + this.numBackticks = numBackticks; + } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Document.java b/commonmark/src/main/java/org/commonmark/node/Document.java index 5b7e74189..6c27b6f2d 100644 --- a/commonmark/src/main/java/org/commonmark/node/Document.java +++ b/commonmark/src/main/java/org/commonmark/node/Document.java @@ -1,9 +1,39 @@ package org.commonmark.node; public class Document extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Emphasis.java b/commonmark/src/main/java/org/commonmark/node/Emphasis.java index 9877e7b63..54acaf7c5 100644 --- a/commonmark/src/main/java/org/commonmark/node/Emphasis.java +++ b/commonmark/src/main/java/org/commonmark/node/Emphasis.java @@ -3,6 +3,7 @@ public class Emphasis extends Node implements Delimited { private String delimiter; + private String preBlockWhitespace = ""; public Emphasis() { } @@ -10,6 +11,15 @@ public Emphasis() { public Emphasis(String delimiter) { this.delimiter = delimiter; } + + public Emphasis(String delimiter, String preBlockWhitespace) { + this(delimiter); + this.preBlockWhitespace = preBlockWhitespace; + } + + public String whitespacePreBlock() { + return preBlockWhitespace; + } public void setDelimiter(String delimiter) { this.delimiter = delimiter; diff --git a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java index 7e2612331..2c55570f3 100644 --- a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java @@ -1,13 +1,20 @@ package org.commonmark.node; public class FencedCodeBlock extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; private char fenceChar; - private int fenceLength; - private int fenceIndent; + private int startFenceLength; + private int endFenceLength; private String info; private String literal; + private String raw; @Override public void accept(Visitor visitor) { @@ -22,20 +29,28 @@ public void setFenceChar(char fenceChar) { this.fenceChar = fenceChar; } - public int getFenceLength() { - return fenceLength; + public int getStartFenceLength() { + return startFenceLength; } - public void setFenceLength(int fenceLength) { - this.fenceLength = fenceLength; + public void setStartFenceLength(int startFenceLength) { + this.startFenceLength = startFenceLength; } - public int getFenceIndent() { - return fenceIndent; + public int getEndFenceLength() { + return endFenceLength; } - public void setFenceIndent(int fenceIndent) { - this.fenceIndent = fenceIndent; + public void setEndFenceLength(int endFenceLength) { + this.endFenceLength = endFenceLength; + } + + public int getStartFenceIndent() { + return whitespaceTracker[0].length(); + } + + public int getEndFenceIndent() { + return whitespaceTracker[2].length(); } /** @@ -56,4 +71,36 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java b/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java index 0640fc3c4..72e9ad8ad 100644 --- a/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java +++ b/commonmark/src/main/java/org/commonmark/node/HardLineBreak.java @@ -1,9 +1,23 @@ package org.commonmark.node; public class HardLineBreak extends Node { - + boolean hasBackslash = false; + + // Preserve original default constructor by explicitly defining one + public HardLineBreak() { + super(); + } + + public HardLineBreak(boolean hasBackslash) { + this.hasBackslash = hasBackslash; + } + @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public boolean hasBackslash() { + return hasBackslash; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Heading.java b/commonmark/src/main/java/org/commonmark/node/Heading.java index 41f3b2504..5ffb060b9 100644 --- a/commonmark/src/main/java/org/commonmark/node/Heading.java +++ b/commonmark/src/main/java/org/commonmark/node/Heading.java @@ -1,8 +1,16 @@ package org.commonmark.node; public class Heading extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; private int level; + private char symbolType; + private int numEndingSymbol; @Override public void accept(Visitor visitor) { @@ -16,4 +24,44 @@ public int getLevel() { public void setLevel(int level) { this.level = level; } + + public char getSymbolType() { + return symbolType; + } + + public void setSymbolType(char symbolType) { + this.symbolType = symbolType; + } + + public int getNumEndingSymbol() { + return numEndingSymbol; + } + + public void setNumEndingSymbol(int numEndingSymbol) { + this.numEndingSymbol = numEndingSymbol; + } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java index ad46c56ce..96ce69f82 100644 --- a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java @@ -6,8 +6,15 @@ * @see CommonMark Spec */ public class HtmlBlock extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; private String literal; + private String raw; @Override public void accept(Visitor visitor) { @@ -21,4 +28,36 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/HtmlInline.java b/commonmark/src/main/java/org/commonmark/node/HtmlInline.java index 291fcde3c..86caf9ce1 100644 --- a/commonmark/src/main/java/org/commonmark/node/HtmlInline.java +++ b/commonmark/src/main/java/org/commonmark/node/HtmlInline.java @@ -8,6 +8,7 @@ public class HtmlInline extends Node { private String literal; + private String raw; @Override public void accept(Visitor visitor) { @@ -21,4 +22,12 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Image.java b/commonmark/src/main/java/org/commonmark/node/Image.java index 63481773a..74bf313e7 100644 --- a/commonmark/src/main/java/org/commonmark/node/Image.java +++ b/commonmark/src/main/java/org/commonmark/node/Image.java @@ -1,16 +1,50 @@ package org.commonmark.node; -public class Image extends Node { +public class Image extends Node implements LinkFormat { - private String destination; + private String destination; + private String rawDestination; private String title; + private String rawTitle; + private String label; + private String rawLabel; + private LinkType linkType; + private char titleSymbol; + private String whitespacePreDestination; + private String whitespacePreTitle; + private String whitespacePostContent; public Image() { } public Image(String destination, String title) { this.destination = destination; + this.rawDestination = ""; this.title = title; + this.rawTitle = ""; + this.label = null; + this.rawLabel = null; + linkType = LinkType.NULL; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; + } + + public Image(String destination, String rawDestination, String title, + String rawTitle, String label, String rawLabel, LinkType linkType, + char titleSymbol, String whitespacePreDestination, + String whitespacePreTitle, String whitespacePostContent) { + this.destination = destination; + this.rawDestination = rawDestination; + this.title = title; + this.rawTitle = rawTitle; + this.label = label; + this.rawLabel = rawLabel; + this.linkType = linkType; + this.titleSymbol = titleSymbol; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostContent = whitespacePostContent; } @Override @@ -18,21 +52,115 @@ public void accept(Visitor visitor) { visitor.visit(this); } + @Override public String getDestination() { return destination; } + @Override public void setDestination(String destination) { this.destination = destination; } + + @Override + public String getRawDestination() { + return rawDestination; + } + @Override + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + + @Override public String getTitle() { return title; } + @Override public void setTitle(String title) { this.title = title; } + + @Override + public String getRawTitle() { + return rawTitle; + } + + @Override + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + @Override + public LinkType getLinkType() { + return linkType; + } + + @Override + public void setLinkType(LinkType linkType) { + this.linkType = linkType; + } + + @Override + public char getTitleSymbol() { + return titleSymbol; + } + + @Override + public void setTitleSymbol(char titleSymbol) { + this.titleSymbol = titleSymbol; + } + + @Override + public String getWhitespacePreDestination() { + return whitespacePreDestination; + } + + @Override + public void setWhitespacePreDestination(String whitespacePreDestination) { + this.whitespacePreDestination = whitespacePreDestination; + } + + @Override + public String getWhitespacePreTitle() { + return whitespacePreTitle; + } + + @Override + public void setWhitespacePreTitle(String whitespacePreTitle) { + this.whitespacePreTitle = whitespacePreTitle; + } + + @Override + public String getWhitespacePostContent() { + return whitespacePostContent; + } + + @Override + public void setWhitespacePostContent(String whitespacePostContent) { + this.whitespacePostContent = whitespacePostContent; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public void setLabel(String label) { + this.label = label; + } + + @Override + public String getRawLabel() { + return rawLabel; + } + + @Override + public void setRawLabel(String rawLabel) { + this.rawLabel = rawLabel; + } @Override protected String toStringAttributes() { diff --git a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java index ccafca943..fb6f599ed 100644 --- a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java @@ -1,8 +1,20 @@ package org.commonmark.node; public class IndentedCodeBlock extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; + // The "literal" string is optimized for HTML formatting, but omits some details + // which are important for roundtrip rendering private String literal; + + // The "raw" string is, as much as possible, the entire raw content as it was first + // entered into the parser. This allows roundtrip parsing for indented code blocks. + private String raw; @Override public void accept(Visitor visitor) { @@ -16,4 +28,36 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Link.java b/commonmark/src/main/java/org/commonmark/node/Link.java index b2ed8c2a1..0622453e1 100644 --- a/commonmark/src/main/java/org/commonmark/node/Link.java +++ b/commonmark/src/main/java/org/commonmark/node/Link.java @@ -20,17 +20,51 @@ * * @see CommonMark Spec for links */ -public class Link extends Node { +public class Link extends Node implements LinkFormat { - private String destination; + private String destination; + private String rawDestination; private String title; + private String rawTitle; + private String label; + private String rawLabel; + private LinkType linkType; + private char titleSymbol; + private String whitespacePreDestination; + private String whitespacePreTitle; + private String whitespacePostContent; public Link() { } public Link(String destination, String title) { this.destination = destination; + this.rawDestination = ""; this.title = title; + this.rawTitle = ""; + this.label = ""; + this.rawLabel = ""; + linkType = LinkType.NULL; + whitespacePreDestination = ""; + whitespacePreTitle = ""; + whitespacePostContent = ""; + } + + public Link(String destination, String rawDestination, String title, + String rawTitle, String label, String rawLabel, + LinkType linkType, char titleSymbol, String whitespacePreDestination, + String whitespacePreTitle, String whitespacePostContent) { + this.destination = destination; + this.rawDestination = rawDestination; + this.title = title; + this.rawTitle = rawTitle; + this.label = label; + this.rawLabel = rawLabel; + this.linkType = linkType; + this.titleSymbol = titleSymbol; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostContent = whitespacePostContent; } @Override @@ -38,22 +72,116 @@ public void accept(Visitor visitor) { visitor.visit(this); } + @Override public String getDestination() { return destination; } + @Override public void setDestination(String destination) { this.destination = destination; } + + @Override + public String getRawDestination() { + return rawDestination; + } + @Override + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + + @Override public String getTitle() { return title; } + @Override public void setTitle(String title) { this.title = title; } + + @Override + public String getRawTitle() { + return rawTitle; + } + + @Override + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + @Override + public LinkType getLinkType() { + return linkType; + } + + @Override + public void setLinkType(LinkType linkType) { + this.linkType = linkType; + } + @Override + public char getTitleSymbol() { + return titleSymbol; + } + + @Override + public void setTitleSymbol(char titleSymbol) { + this.titleSymbol = titleSymbol; + } + + @Override + public String getWhitespacePreDestination() { + return whitespacePreDestination; + } + + @Override + public void setWhitespacePreDestination(String whitespacePreDestination) { + this.whitespacePreDestination = whitespacePreDestination; + } + + @Override + public String getWhitespacePreTitle() { + return whitespacePreTitle; + } + + @Override + public void setWhitespacePreTitle(String whitespacePreTitle) { + this.whitespacePreTitle = whitespacePreTitle; + } + + @Override + public String getWhitespacePostContent() { + return whitespacePostContent; + } + + @Override + public void setWhitespacePostContent(String whitespacePostContent) { + this.whitespacePostContent = whitespacePostContent; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public void setLabel(String label) { + this.label = label; + } + + @Override + public String getRawLabel() { + return rawLabel; + } + + @Override + public void setRawLabel(String rawLabel) { + this.rawLabel = rawLabel; + } + @Override protected String toStringAttributes() { return "destination=" + destination + ", title=" + title; diff --git a/commonmark/src/main/java/org/commonmark/node/LinkFormat.java b/commonmark/src/main/java/org/commonmark/node/LinkFormat.java new file mode 100644 index 000000000..267bcfa3d --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/node/LinkFormat.java @@ -0,0 +1,30 @@ +package org.commonmark.node; + +public interface LinkFormat { + public String getDestination(); + public void setDestination(String destination); + public String getRawDestination(); + public void setRawDestination(String destination); + public String getLabel(); + public void setLabel(String label); + public String getRawLabel(); + public void setRawLabel(String label); + public String getTitle(); + public void setTitle(String title); + public String getRawTitle(); + public void setRawTitle(String title); + public char getTitleSymbol(); + public void setTitleSymbol(char symbol); + public LinkType getLinkType(); + public void setLinkType(LinkType linkType); + public String getWhitespacePreDestination(); + public void setWhitespacePreDestination(String whitespace); + public String getWhitespacePreTitle(); + public void setWhitespacePreTitle(String whitespace); + public String getWhitespacePostContent(); + public void setWhitespacePostContent(String whitespace); + + public enum LinkType { + INLINE, REFERENCE, AUTOLINK, NULL; + } +} diff --git a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java index 3f8bfd0f0..c8e4d3b12 100644 --- a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java +++ b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java @@ -13,17 +13,31 @@ */ public class LinkReferenceDefinition extends Node { - private String label; + private String label; private String destination; + private String rawDestination; private String title; + private String rawTitle; + private char delimiterChar; + private String whitespacePreLabel = ""; + private String whitespacePreDestination = ""; + private String whitespacePreTitle = ""; + private String whitespacePostTitle = ""; public LinkReferenceDefinition() { } - public LinkReferenceDefinition(String label, String destination, String title) { + public LinkReferenceDefinition(String label, String destination, String rawDestination, String title, String rawTitle, char delimiterChar, String whitespacePreLabel, String whitespacePreDestination, String whitespacePreTitle, String whitespacePostTitle) { this.label = label; this.destination = destination; + this.rawDestination = rawDestination; this.title = title; + this.rawTitle = rawTitle; + this.delimiterChar = delimiterChar; + this.whitespacePreLabel = whitespacePreLabel; + this.whitespacePreDestination = whitespacePreDestination; + this.whitespacePreTitle = whitespacePreTitle; + this.whitespacePostTitle = whitespacePostTitle; } public String getLabel() { @@ -33,7 +47,7 @@ public String getLabel() { public void setLabel(String label) { this.label = label; } - + public String getDestination() { return destination; } @@ -42,6 +56,14 @@ public void setDestination(String destination) { this.destination = destination; } + public String getRawDestination() { + return rawDestination; + } + + public void setRawDestination(String rawDestination) { + this.rawDestination = rawDestination; + } + public String getTitle() { return title; } @@ -49,6 +71,34 @@ public String getTitle() { public void setTitle(String title) { this.title = title; } + + public String getRawTitle() { + return rawTitle; + } + + public void setRawTitle(String rawTitle) { + this.rawTitle = rawTitle; + } + + public char getDelimiterChar() { + return delimiterChar; + } + + public String whitespacePreLabel() { + return whitespacePreLabel; + } + + public String whitespacePreDestination() { + return whitespacePreDestination; + } + + public String whitespacePreTitle() { + return whitespacePreTitle; + } + + public String whitespacePostTitle() { + return whitespacePostTitle; + } @Override public void accept(Visitor visitor) { diff --git a/commonmark/src/main/java/org/commonmark/node/ListItem.java b/commonmark/src/main/java/org/commonmark/node/ListItem.java index aa526be01..de551b918 100644 --- a/commonmark/src/main/java/org/commonmark/node/ListItem.java +++ b/commonmark/src/main/java/org/commonmark/node/ListItem.java @@ -1,9 +1,48 @@ package org.commonmark.node; public class ListItem extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; + private String rawNumber = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } + + public String getRawNumber() { + return rawNumber; + } + + public void setRawNumber(String rawNumber) { + this.rawNumber = rawNumber; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Node.java b/commonmark/src/main/java/org/commonmark/node/Node.java index 5a2f036e4..39bdc2e5b 100644 --- a/commonmark/src/main/java/org/commonmark/node/Node.java +++ b/commonmark/src/main/java/org/commonmark/node/Node.java @@ -150,6 +150,21 @@ public void addSourceSpan(SourceSpan sourceSpan) { this.sourceSpans.add(sourceSpan); } + /** + * Check whether the given node is of the subset which count as "inline" nodes + * @param node Node to be checked for inline + * @return true if node is an inline node, otherwise false + */ + public static boolean isInlineNode(Node node) { + if(node instanceof Code || node instanceof Emphasis || node instanceof StrongEmphasis || + node instanceof Link || node instanceof Image || node instanceof HtmlInline || + node instanceof HardLineBreak) { + return true; + }else { + return false; + } + } + @Override public String toString() { return getClass().getSimpleName() + "{" + toStringAttributes() + "}"; diff --git a/commonmark/src/main/java/org/commonmark/node/OrderedList.java b/commonmark/src/main/java/org/commonmark/node/OrderedList.java index 1f988234c..560453836 100644 --- a/commonmark/src/main/java/org/commonmark/node/OrderedList.java +++ b/commonmark/src/main/java/org/commonmark/node/OrderedList.java @@ -1,8 +1,15 @@ package org.commonmark.node; public class OrderedList extends ListBlock { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; private int startNumber; + private String rawNumber; private char delimiter; @Override @@ -17,6 +24,14 @@ public int getStartNumber() { public void setStartNumber(int startNumber) { this.startNumber = startNumber; } + + public String getRawNumber() { + return rawNumber; + } + + public void setRawNumber(String rawNumber) { + this.rawNumber = rawNumber; + } public char getDelimiter() { return delimiter; @@ -26,4 +41,27 @@ public void setDelimiter(char delimiter) { this.delimiter = delimiter; } + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Paragraph.java b/commonmark/src/main/java/org/commonmark/node/Paragraph.java index 176eaaa76..e3e08e308 100644 --- a/commonmark/src/main/java/org/commonmark/node/Paragraph.java +++ b/commonmark/src/main/java/org/commonmark/node/Paragraph.java @@ -4,9 +4,39 @@ * A paragraph block, contains inline nodes such as {@link Text} */ public class Paragraph extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java b/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java index dbff571cd..939944dd8 100644 --- a/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java +++ b/commonmark/src/main/java/org/commonmark/node/StrongEmphasis.java @@ -4,12 +4,24 @@ public class StrongEmphasis extends Node implements Delimited { private String delimiter; + // Specifically used during roundtrip rendering of thematic breaks + private String preBlockWhitespace; + public StrongEmphasis() { } public StrongEmphasis(String delimiter) { this.delimiter = delimiter; } + + public StrongEmphasis(String delimiter, String preBlockWhitespace) { + this(delimiter); + this.preBlockWhitespace = preBlockWhitespace; + } + + public String whitespacePreBlock() { + return preBlockWhitespace; + } public void setDelimiter(String delimiter) { this.delimiter = delimiter; diff --git a/commonmark/src/main/java/org/commonmark/node/Text.java b/commonmark/src/main/java/org/commonmark/node/Text.java index f16fc907b..0b227af91 100644 --- a/commonmark/src/main/java/org/commonmark/node/Text.java +++ b/commonmark/src/main/java/org/commonmark/node/Text.java @@ -1,14 +1,29 @@ package org.commonmark.node; +import java.util.Objects; + public class Text extends Node { - private String literal; + private String literal; + private String raw = ""; + private String preContentWhitespace; + private String postContentWhitespace; public Text() { } public Text(String literal) { this.literal = literal; + this.raw = literal; + this.preContentWhitespace = ""; + this.postContentWhitespace = ""; + } + + public Text(String literal, String raw, String preContentWhitespace, String postContentWhitespace) { + this.literal = literal; + this.raw = raw; + this.preContentWhitespace = preContentWhitespace; + this.postContentWhitespace = postContentWhitespace; } @Override @@ -23,9 +38,56 @@ public String getLiteral() { public void setLiteral(String literal) { this.literal = literal; } + + public String getRaw() { + return raw; + } + + public void setRaw(String raw) { + this.raw = raw; + } + + public String whitespacePreContent() { + if(preContentWhitespace != null) { + return preContentWhitespace; + }else { + return ""; + } + } + + public String whitespacePostContent() { + if(postContentWhitespace != null) { + return postContentWhitespace; + }else { + return ""; + } + } + + public void setWhitespace(String pre, String post) { + preContentWhitespace = pre; + postContentWhitespace = post; + } @Override protected String toStringAttributes() { return "literal=" + literal; } + + @Override + public boolean equals(Object obj) { + if(obj instanceof Text) { + if(((Text)obj).getLiteral().equals(literal) && ((Text)obj).getRaw().equals(raw) && + ((Text)obj).whitespacePreContent().equals(preContentWhitespace) && + ((Text)obj).whitespacePostContent().equals(postContentWhitespace)) { + return true; + } + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hash(literal, raw, preContentWhitespace, postContentWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java index f81abaa31..5be26fef0 100644 --- a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java +++ b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java @@ -1,9 +1,49 @@ package org.commonmark.node; public class ThematicBreak extends Block { + // Track whitespace as follows: + // [0] Pre-block + // [1] Pre-content + // [2] Post-content + // [3] Post-block + private String[] whitespaceTracker = {"", "", "", ""}; + + private CharSequence content; @Override public void accept(Visitor visitor) { visitor.visit(this); } + + public CharSequence getContent() { + return content; + } + + public void setContent(CharSequence content) { + this.content = content; + } + + @Override + public String whitespacePreBlock() { + return whitespaceTracker[0]; + } + + @Override + public String whitespacePreContent() { + return whitespaceTracker[1]; + } + + @Override + public String whitespacePostContent() { + return whitespaceTracker[2]; + } + + @Override + public String whitespacePostBlock() { + return whitespaceTracker[3]; + } + + public void setWhitespace(String... newWhitespace) { + whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Visitor.java b/commonmark/src/main/java/org/commonmark/node/Visitor.java index a155296f0..9df964c35 100644 --- a/commonmark/src/main/java/org/commonmark/node/Visitor.java +++ b/commonmark/src/main/java/org/commonmark/node/Visitor.java @@ -7,6 +7,8 @@ */ public interface Visitor { + void visit(BlankLine blankLine); + void visit(BlockQuote blockQuote); void visit(BulletList bulletList); diff --git a/commonmark/src/main/java/org/commonmark/parser/SourceLine.java b/commonmark/src/main/java/org/commonmark/parser/SourceLine.java index 63caceb9e..bdbb3dbda 100644 --- a/commonmark/src/main/java/org/commonmark/parser/SourceLine.java +++ b/commonmark/src/main/java/org/commonmark/parser/SourceLine.java @@ -1,5 +1,7 @@ package org.commonmark.parser; +import java.util.Objects; + import org.commonmark.node.SourceSpan; /** @@ -11,10 +13,16 @@ public class SourceLine { private final CharSequence content; private final SourceSpan sourceSpan; + private final int literalIndex; public static SourceLine of(CharSequence content, SourceSpan sourceSpan) { return new SourceLine(content, sourceSpan); } + + // The literal index is the beginning of a literal line, as found within a raw line + public static SourceLine of(CharSequence content, SourceSpan sourceSpan, int literalIndex) { + return new SourceLine(content, sourceSpan, literalIndex); + } private SourceLine(CharSequence content, SourceSpan sourceSpan) { if (content == null) { @@ -22,6 +30,16 @@ private SourceLine(CharSequence content, SourceSpan sourceSpan) { } this.content = content; this.sourceSpan = sourceSpan; + this.literalIndex = 0; + } + + private SourceLine(CharSequence content, SourceSpan sourceSpan, int literalIndex) { + if (content == null) { + throw new NullPointerException("content must not be null"); + } + this.content = content; + this.sourceSpan = sourceSpan; + this.literalIndex = literalIndex; } public CharSequence getContent() { @@ -31,6 +49,10 @@ public CharSequence getContent() { public SourceSpan getSourceSpan() { return sourceSpan; } + + public int getLiteralIndex() { + return literalIndex; + } public SourceLine substring(int beginIndex, int endIndex) { CharSequence newContent = content.subSequence(beginIndex, endIndex); @@ -44,4 +66,41 @@ public SourceLine substring(int beginIndex, int endIndex) { } return SourceLine.of(newContent, newSourceSpan); } + + // Many lines are passed around as raw strings, with an index value indicating + // where the literal string begins. Calling this method returns a SourceLine containing + // only the literal portion of the line (if it is a subset of the line) or the line + // in full (if they are the same). + public SourceLine getLiteralLine() { + if(literalIndex == 0) { + return SourceLine.of(content, sourceSpan); + }else { + SourceSpan newSourceSpan = null; + if(sourceSpan != null) { + int columnIndex = sourceSpan.getColumnIndex() - literalIndex; + int length = content.length() - literalIndex; + if(length != 0) { + newSourceSpan = SourceSpan.of(sourceSpan.getLineIndex(), columnIndex, length); + } + } + return SourceLine.of(content.subSequence(literalIndex, content.length()), newSourceSpan); + } + } + + @Override + public int hashCode() { + return Objects.hash(content, literalIndex); + } + + @Override + public boolean equals(Object obj) { + if(obj instanceof SourceLine) { + if(((SourceLine)obj).getContent().toString().equals(content) && + ((SourceLine)obj).getLiteralIndex() == literalIndex) { + return true; + } + } + + return false; + } } diff --git a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java index 0b4290341..618fd044b 100644 --- a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java +++ b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java @@ -1,9 +1,10 @@ package org.commonmark.parser; -import org.commonmark.node.SourceSpan; - import java.util.ArrayList; import java.util.List; +import java.util.Objects; + +import org.commonmark.node.SourceSpan; /** * A set of lines ({@link SourceLine}) from the input source. @@ -63,4 +64,32 @@ public List getSourceSpans() { } return sourceSpans; } + + /** + * Return original 
List<SourceLine&rt;
, but with first line removed. + * If the list is already empty, return the list unchanged + */ + public List removeFirstLine() { + if(!isEmpty()) { + lines.remove(0); + } + + return lines; + } + + @Override + public int hashCode() { + return Objects.hash(lines); + } + + @Override + public boolean equals(Object obj) { + if(obj instanceof SourceLines) { + if(((SourceLines)obj).getLines().equals(lines)) { + return true; + } + } + + return false; + } } diff --git a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java index 897943d66..c1f3663e1 100644 --- a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java @@ -41,4 +41,19 @@ public interface DelimiterProcessor { */ int process(DelimiterRun openingRun, DelimiterRun closingRun); + /** + * Process the delimiter runs. + *

+ * The processor can examine the runs and the nodes and decide if it wants to process or not. If not, it should not + * change any nodes and return 0. If yes, it should do the processing (wrapping nodes, etc) and then return how many + * delimiters were used. + *

+ * Note that removal (unlinking) of the used delimiter {@link Text} nodes is done by the caller. + * + * @param openingRun the opening delimiter run + * @param closingRun the closing delimiter run + * @param prefix whitespace before the delimiter run, if any + * @return how many delimiters were used; must not be greater than length of either opener or closer + */ + int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix); } diff --git a/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java index 7d3552668..54756895f 100644 --- a/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java +++ b/commonmark/src/main/java/org/commonmark/renderer/html/CoreHtmlNodeRenderer.java @@ -1,10 +1,39 @@ package org.commonmark.renderer.html; -import org.commonmark.node.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.commonmark.internal.util.Escaping; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.BlankLine; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Document; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; import org.commonmark.renderer.NodeRenderer; -import java.util.*; - /** * The node renderer that renders all the core nodes (comes last in the order of node renderers). */ @@ -40,7 +69,8 @@ public Set> getNodeTypes() { Code.class, HtmlInline.class, SoftLineBreak.class, - HardLineBreak.class + HardLineBreak.class, + BlankLine.class )); } @@ -99,7 +129,11 @@ public void visit(BulletList bulletList) { public void visit(FencedCodeBlock fencedCodeBlock) { String literal = fencedCodeBlock.getLiteral(); Map attributes = new LinkedHashMap<>(); - String info = fencedCodeBlock.getInfo(); + + // The info string for FencedCodeBlock is not escaped or trimmed + // of excess whitespace, do that now + String info = Escaping.unescapeString(fencedCodeBlock.getInfo()).trim(); + if (info != null && !info.isEmpty()) { int space = info.indexOf(" "); String language; @@ -244,6 +278,13 @@ public void visit(HardLineBreak hardLineBreak) { html.line(); } + @Override + public void visit(BlankLine blankLine) { + // The BlankLine class is not part of the CommonMark standard, and is + // only for roundtrip rendering. Thus, it is specifically ignored + // by the HTML renderer. + } + @Override protected void visitChildren(Node parent) { Node node = parent.getFirstChild(); diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java new file mode 100644 index 000000000..bf0778464 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkNodeRendererContext.java @@ -0,0 +1,25 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.node.Node; + +public interface CommonMarkNodeRendererContext { + + /** + * @return true for stripping new lines and render text as "single line", + * false for keeping all line breaks. + */ + boolean stripNewlines(); + + /** + * @return the writer to use + */ + CommonMarkWriter getWriter(); + + /** + * Render the specified node and its children using the configured renderers. This should be used to render child + * nodes; be careful not to pass the node that is being rendered, that would result in an endless loop. + * + * @param node the node to render + */ + void render(Node node); +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java new file mode 100644 index 000000000..d24336334 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkRenderer.java @@ -0,0 +1,149 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.Extension; +import org.commonmark.internal.renderer.NodeRendererMap; +import org.commonmark.node.Node; +import org.commonmark.renderer.NodeRenderer; +import org.commonmark.renderer.Renderer; + +import java.util.ArrayList; +import java.util.List; + +public class CommonMarkRenderer implements Renderer { + + private final boolean stripNewlines; + + private final List nodeRendererFactories; + + private CommonMarkRenderer(Builder builder) { + this.stripNewlines = builder.stripNewlines; + + this.nodeRendererFactories = new ArrayList<>(builder.nodeRendererFactories.size() + 1); + this.nodeRendererFactories.addAll(builder.nodeRendererFactories); + // Add as last. This means clients can override the rendering of core nodes if they want. + this.nodeRendererFactories.add(new CommonmarkNodeRendererFactory() { + @Override + public NodeRenderer create(CommonMarkNodeRendererContext context) { + return new CoreCommonMarkNodeRenderer(context); + } + }); + } + + /** + * Create a new builder for configuring an {@link CommonMarkRenderer}. + * + * @return a builder + */ + public static Builder builder() { + return new Builder(); + } + + @Override + public void render(Node node, Appendable output) { + RendererContext context = new RendererContext(new CommonMarkWriter(output)); + context.render(node); + } + + @Override + public String render(Node node) { + StringBuilder sb = new StringBuilder(); + render(node, sb); + return sb.toString(); + } + + /** + * Builder for configuring an {@link CommonMarkRenderer}. See methods for default configuration. + */ + public static class Builder { + + private boolean stripNewlines = false; + private List nodeRendererFactories = new ArrayList<>(); + + /** + * @return the configured {@link CommonMarkRenderer} + */ + public CommonMarkRenderer build() { + return new CommonMarkRenderer(this); + } + + /** + * Set the value of flag for stripping new lines. + * + * @param stripNewlines true for stripping new lines and render text as "single line", + * false for keeping all line breaks + * @return {@code this} + */ + public Builder stripNewlines(boolean stripNewlines) { + this.stripNewlines = stripNewlines; + return this; + } + + /** + * Add a factory for instantiating a node renderer (done when rendering). This allows to override the rendering + * of node types or define rendering for custom node types. + *

+ * If multiple node renderers for the same node type are created, the one from the factory that was added first + * "wins". (This is how the rendering for core node types can be overridden; the default rendering comes last.) + * + * @param nodeRendererFactory the factory for creating a node renderer + * @return {@code this} + */ + public Builder nodeRendererFactory(CommonmarkNodeRendererFactory nodeRendererFactory) { + this.nodeRendererFactories.add(nodeRendererFactory); + return this; + } + + /** + * @param extensions extensions to use on this text content renderer + * @return {@code this} + */ + public Builder extensions(Iterable extensions) { + for (Extension extension : extensions) { + if (extension instanceof CommonMarkRenderer.CommonMarkRendererExtension) { + CommonMarkRenderer.CommonMarkRendererExtension textContentRendererExtension = + (CommonMarkRenderer.CommonMarkRendererExtension) extension; + textContentRendererExtension.extend(this); + } + } + return this; + } + } + + /** + * Extension for {@link CommonMarkRenderer}. + */ + public interface CommonMarkRendererExtension extends Extension { + void extend(CommonMarkRenderer.Builder rendererBuilder); + } + + private class RendererContext implements CommonMarkNodeRendererContext { + private final CommonMarkWriter commonMarkWriter; + private final NodeRendererMap nodeRendererMap = new NodeRendererMap(); + + private RendererContext(CommonMarkWriter commonMarkWriter) { + this.commonMarkWriter = commonMarkWriter; + + // The first node renderer for a node type "wins". + for (int i = nodeRendererFactories.size() - 1; i >= 0; i--) { + CommonmarkNodeRendererFactory nodeRendererFactory = nodeRendererFactories.get(i); + NodeRenderer nodeRenderer = nodeRendererFactory.create(this); + nodeRendererMap.add(nodeRenderer); + } + } + + @Override + public boolean stripNewlines() { + return stripNewlines; + } + + @Override + public CommonMarkWriter getWriter() { + return commonMarkWriter; + } + + @Override + public void render(Node node) { + nodeRendererMap.render(node); + } + } +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java new file mode 100644 index 000000000..8fc5b683a --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java @@ -0,0 +1,65 @@ +package org.commonmark.renderer.roundtrip; + +import java.io.IOException; + +public class CommonMarkWriter { + + private final Appendable buffer; + + private char lastChar; + + public CommonMarkWriter(Appendable out) { + buffer = out; + } + + public void whitespace() { + if (lastChar != 0 && lastChar != ' ') { + append(' '); + } + } + + public void colon() { + if (lastChar != 0 && lastChar != ':') { + append(':'); + } + } + + public void line() { + append('\n'); + } + + public void writeStripped(String s) { + append(s.replaceAll("[\\r\\n\\s]+", " ")); + } + + public void write(String s) { + append(s); + } + + public void write(char c) { + append(c); + } + + private void append(String s) { + try { + buffer.append(s); + } catch (IOException e) { + throw new RuntimeException(e); + } + + int length = s.length(); + if (length != 0) { + lastChar = s.charAt(length - 1); + } + } + + private void append(char c) { + try { + buffer.append(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + + lastChar = c; + } +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java new file mode 100644 index 000000000..9c3bd61f4 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonmarkNodeRendererFactory.java @@ -0,0 +1,17 @@ +package org.commonmark.renderer.roundtrip; + +import org.commonmark.renderer.NodeRenderer; + +/** + * Factory for instantiating new node renderers when rendering is done. + */ +public interface CommonmarkNodeRendererFactory { + + /** + * Create a new node renderer for the specified rendering context. + * + * @param context the context for rendering (normally passed on to the node renderer) + * @return a node renderer + */ + NodeRenderer create(CommonMarkNodeRendererContext context); +} diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java new file mode 100644 index 000000000..187a3c5ff --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java @@ -0,0 +1,575 @@ +package org.commonmark.renderer.roundtrip; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.commonmark.internal.renderer.text.BulletListHolder; +import org.commonmark.internal.renderer.text.ListHolder; +import org.commonmark.internal.renderer.text.OrderedListHolder; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.BlankLine; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Document; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.LinkFormat.LinkType; +import org.commonmark.node.LinkReferenceDefinition; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.renderer.NodeRenderer; + +/** + * The node renderer that renders all the core nodes (comes last in the order of node renderers). + */ +public class CoreCommonMarkNodeRenderer extends AbstractVisitor implements NodeRenderer { + + protected final CommonMarkNodeRendererContext context; + private final CommonMarkWriter textContent; + + private ListHolder listHolder; + + public CoreCommonMarkNodeRenderer(CommonMarkNodeRendererContext context) { + this.context = context; + this.textContent = context.getWriter(); + } + + @Override + public Set> getNodeTypes() { + return new HashSet<>(Arrays.asList( + Document.class, + Heading.class, + Paragraph.class, + BlockQuote.class, + BulletList.class, + FencedCodeBlock.class, + HtmlBlock.class, + ThematicBreak.class, + IndentedCodeBlock.class, + Link.class, + LinkReferenceDefinition.class, + ListBlock.class, + ListItem.class, + OrderedList.class, + Image.class, + Emphasis.class, + StrongEmphasis.class, + Text.class, + Code.class, + HtmlInline.class, + SoftLineBreak.class, + HardLineBreak.class, + BlankLine.class + )); + } + + @Override + public void render(Node node) { + node.accept(this); + } + + @Override + public void visit(Document document) { + // No rendering itself (aside from post-block whitespace, if present) + + visitChildren(document); + + textContent.write(document.whitespacePostBlock()); + } + + public void visit(BlockQuote blockQuote) { + if(blockQuote.getFirstChild() instanceof Heading || + blockQuote.getFirstChild() instanceof IndentedCodeBlock || + blockQuote.getFirstChild() instanceof FencedCodeBlock) { + textContent.write(blockQuote.whitespacePreBlock()); + textContent.write(">"); + textContent.write(blockQuote.whitespacePreContent()); + } + + visitChildren(blockQuote); + + writeEndOfLineIfNeeded(blockQuote, null); + } + + @Override + public void visit(BulletList bulletList) { + listHolder = new BulletListHolder(listHolder, bulletList); + visitChildren(bulletList); + writeEndOfLineIfNeeded(bulletList, null); + + if (listHolder.getParent() != null) { + listHolder = listHolder.getParent(); + } else { + listHolder = null; + } + } + + public void visit(Code code) { + for(int i = 0; i < code.getNumBackticks(); i++) { + textContent.write('`'); + } + + textContent.write(code.getRaw()); + + for(int i = 0; i < code.getNumBackticks(); i++) { + textContent.write('`'); + } + } + + @Override + public void visit(Emphasis emphasis) { + textContent.write(emphasis.whitespacePreBlock()); + textContent.write(emphasis.getOpeningDelimiter()); + visitChildren(emphasis); + textContent.write(emphasis.getClosingDelimiter()); + } + + public void visit(FencedCodeBlock fencedCodeBlock) { + for(int i = 0; i < fencedCodeBlock.getStartFenceIndent(); i++) { + textContent.write(" "); + } + + for(int i = 0; i < fencedCodeBlock.getStartFenceLength(); i++) { + textContent.write(fencedCodeBlock.getFenceChar()); + } + + if(fencedCodeBlock.getInfo() != null && !fencedCodeBlock.getInfo().isEmpty()) { + textContent.write(fencedCodeBlock.getInfo()); + } + + // CommonMark test case #96 shows that content with no ending fence and + // completely blank content should not trigger a new line + if(fencedCodeBlock.getEndFenceLength() > 0 || !fencedCodeBlock.getRaw().isEmpty()) { + textContent.line(); + } + + textContent.write(fencedCodeBlock.getRaw()); + + if(fencedCodeBlock.getEndFenceLength() > 0 && !fencedCodeBlock.getRaw().isEmpty()) { + textContent.line(); + } + + for(int i = 0; i < fencedCodeBlock.getEndFenceIndent(); i++) { + textContent.write(" "); + } + + for(int i = 0; i < fencedCodeBlock.getEndFenceLength(); i++) { + textContent.write(fencedCodeBlock.getFenceChar()); + } + + if(fencedCodeBlock.getEndFenceLength() > 0) { + textContent.write(fencedCodeBlock.whitespacePostBlock()); + } + + writeEndOfLineIfNeeded(fencedCodeBlock, null); + } + + @Override + public void visit(HardLineBreak hardLineBreak) { + if(hardLineBreak.hasBackslash()) { + textContent.write("\\"); + } + + writeEndOfLine(); + } + + public void visit(Heading heading) { + + if(heading.getSymbolType() == '#') { + textContent.write(heading.whitespacePreBlock()); + + for(int i = 0; i < heading.getLevel(); i++) { + textContent.write(heading.getSymbolType()); + } + + textContent.write(heading.whitespacePreContent()); + + visitChildren(heading); + + textContent.write(heading.whitespacePostContent()); + + if(heading.getNumEndingSymbol() > 0) { + for(int i = 0; i < heading.getNumEndingSymbol(); i++) { + textContent.write(heading.getSymbolType()); + } + } + + textContent.write(heading.whitespacePostBlock()); + + writeEndOfLineIfNeeded(heading, null); + + }else { + textContent.write(heading.whitespacePreBlock()); + + visitChildren(heading); + + textContent.write(heading.whitespacePreContent()); + + writeEndOfLine(); + + textContent.write(heading.whitespacePostContent()); + + if(heading.getNumEndingSymbol() > 0) { + for(int i = 0; i < heading.getNumEndingSymbol(); i++) { + textContent.write(heading.getSymbolType()); + } + } + + textContent.write(heading.whitespacePostBlock()); + + writeEndOfLineIfNeeded(heading, null); + } + } + + public void visit(ThematicBreak thematicBreak) { + if(thematicBreak.getContent() != null) { + textContent.write(thematicBreak.getContent().toString()); + } + + writeEndOfLineIfNeeded(thematicBreak, null); + } + + @Override + public void visit(HtmlInline htmlInline) { + textContent.write(htmlInline.getLiteral()); + } + + @Override + public void visit(HtmlBlock htmlBlock) { + textContent.write(htmlBlock.getRaw()); + writeEndOfLineIfNeeded(htmlBlock, null); + } + + @Override + public void visit(Image image) { + writeLink(image, image.getTitle(), image.getDestination()); + } + + @Override + public void visit(IndentedCodeBlock indentedCodeBlock) { + textContent.write(indentedCodeBlock.whitespacePreContent()); + + textContent.write(indentedCodeBlock.getRaw()); + + writeEndOfLineIfNeeded(indentedCodeBlock, null); + } + + public void visit(Link link) { + writeLink(link, link.getTitle(), link.getDestination()); + } + + @Override + public void visit(LinkReferenceDefinition linkDef) { + writeLinkReference(linkDef); + writeEndOfLineIfNeeded(linkDef, null); + } + + @Override + public void visit(ListItem listItem) { + if (listHolder != null && listHolder instanceof OrderedListHolder) { + OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder; + + textContent.write(listItem.whitespacePreBlock()); + + if(!(listItem.getFirstChild() instanceof ThematicBreak) && + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(listItem.getRawNumber() + ""); + textContent.write(orderedListHolder.getDelimiter()); + textContent.write(listItem.whitespacePreContent()); + } + + visitChildren(listItem); + + textContent.write(listItem.whitespacePostBlock()); + + writeEndOfLineIfNeeded(listItem, null); + } else if (listHolder != null && listHolder instanceof BulletListHolder) { + BulletListHolder bulletListHolder = (BulletListHolder) listHolder; + + textContent.write(listItem.whitespacePreBlock()); + + if(!(listItem.getFirstChild() instanceof ThematicBreak) && + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(bulletListHolder.getMarker()); + textContent.write(listItem.whitespacePreContent()); + } + visitChildren(listItem); + + textContent.write(listItem.whitespacePostBlock()); + + writeEndOfLineIfNeeded(listItem, null); + } + + } + + @Override + public void visit(OrderedList orderedList) { + listHolder = new OrderedListHolder(listHolder, orderedList); + visitChildren(orderedList); + writeEndOfLineIfNeeded(orderedList, null); + + if (listHolder.getParent() != null) { + listHolder = listHolder.getParent(); + } else { + listHolder = null; + } + + } + + public void visit(Paragraph paragraph) { + visitChildren(paragraph); + + writeEndOfLineIfNeeded(paragraph, null); + + textContent.write(paragraph.whitespacePostBlock()); + } + + @Override + public void visit(SoftLineBreak softLineBreak) { + // Within roundtrip rendering, a soft break is always a newline of some kind + writeEndOfLine(); + } + + @Override + public void visit(StrongEmphasis strongEmphasis) { + textContent.write(strongEmphasis.whitespacePreBlock()); + textContent.write(strongEmphasis.getOpeningDelimiter()); + visitChildren(strongEmphasis); + textContent.write(strongEmphasis.getClosingDelimiter()); + } + + @Override + public void visit(Text text) { + if(!text.getRaw().isEmpty()) { + textContent.write(text.whitespacePreContent() + text.getRaw() + text.whitespacePostContent()); + }else { + textContent.write(text.whitespacePreContent() + text.getLiteral() + text.whitespacePostContent()); + } + } + + @Override + public void visit(BlankLine blankLine) { + System.out.println(blankLine.getRaw()); + textContent.write(blankLine.getRaw()); + writeEndOfLineIfNeeded(blankLine, null); + } + + @Override + protected void visitChildren(Node parent) { + Node node = parent.getFirstChild(); + + while (node != null) { + Node next = node.getNext(); + + context.render(node); + node = next; + } + } + + private void writeLink(Node node, String title, String destination) { + if(node instanceof Link) { + writeLink((Link)node); + } + + if(node instanceof Image) { + writeImage((Image)node); + } + } + + private void writeLink(Link node) { + // Autolink + if(node.getLinkType() == LinkType.AUTOLINK) { + textContent.write("<"); + visitChildren(node); + textContent.write(">"); + }else if(node.getLinkType() == LinkType.REFERENCE) { + textContent.write("["); + visitChildren(node); + textContent.write("]"); + + if(node.getLabel() != null) { + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); + } + }else { + textContent.write("["); + visitChildren(node); + textContent.write("]"); + + textContent.write("("); + + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); + }else if(node.getDestination() != null) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getDestination()); + }else { + visitChildren(node); + } + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); + }else if(node.getTitle() != null) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getTitleSymbol() + ""); + + textContent.write(node.getTitle()); + + if(node.getTitleSymbol() == '(') { + textContent.write(")"); + }else { + textContent.write(node.getTitleSymbol()); + } + } + + textContent.write(node.getWhitespacePostContent()); + textContent.write(")"); + } + } + + private void writeLinkReference(LinkReferenceDefinition node) { + textContent.write(node.whitespacePreLabel()); + + textContent.write("["); + textContent.write(node.getLabel()); + textContent.write("]:"); + + if(node.getDestination() != null) { + textContent.write(node.whitespacePreDestination()); + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getRawDestination()); + }else { + textContent.write(node.getDestination()); + } + + if(node.getTitle() != null) { + textContent.write(node.whitespacePreTitle()); + + char delimiter = node.getDelimiterChar(); + + if(delimiter == Character.MIN_VALUE) { + delimiter = '"'; + } + + textContent.write(delimiter); + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getRawTitle()); + }else { + textContent.write(node.getTitle()); + } + + textContent.write(delimiter); + } + } + + textContent.write(node.whitespacePostTitle()); + } + + private void writeImage(Image node) { + // Autolink + if(node.getLinkType() == LinkType.AUTOLINK) { + textContent.write("<"); + visitChildren(node); + textContent.write(">"); + }else if(node.getLinkType() == LinkType.REFERENCE) { + textContent.write("!["); + visitChildren(node); + textContent.write("]"); + + if(node.getLabel() != null) { + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); + } + }else { + textContent.write("!["); + visitChildren(node); + textContent.write("]"); + + textContent.write("("); + + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); + }else if(node.getDestination() != null) { + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getDestination()); + }else { + visitChildren(node); + } + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); + }else if(node.getTitle() != null) { + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getTitleSymbol() + ""); + + textContent.write(node.getTitle()); + + if(node.getTitleSymbol() == '(') { + textContent.write(")"); + }else { + textContent.write(node.getTitleSymbol()); + } + } + + textContent.write(node.getWhitespacePostContent()); + textContent.write(")"); + } + } + + private void writeEndOfLineIfNeeded(Node node, Character c) { + if (node.getNext() != null) { + if(node instanceof ListItem) { + // If a list item has post-block whitespace, allow it to handle whitespace + if(((ListItem)node).whitespacePostBlock().isEmpty()) { + textContent.line(); + } + }else if(node instanceof Paragraph) { + // If a paragraph has post-block whitespace, allow it to handle whitespace + if(((Paragraph)node).whitespacePostBlock().isEmpty()) { + textContent.line(); + } + }else { + textContent.line(); + } + } + } + + private void writeEndOfLine() { + textContent.line(); + } +} diff --git a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java index 173d711d3..7bcc4a963 100644 --- a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java +++ b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java @@ -102,6 +102,11 @@ public int getMinLength() { public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return delimiterUse; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class AsymmetricDelimiterProcessor implements DelimiterProcessor { @@ -136,6 +141,11 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class UpperCaseNode extends CustomNode { @@ -198,6 +208,11 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { closingRun.getCloser().insertBefore(new Text("(/1)")); return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } private static class TwoDelimiterProcessor implements DelimiterProcessor { @@ -223,5 +238,10 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { closingRun.getCloser().insertBefore(new Text("(/2)")); return 2; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } } diff --git a/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java b/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java index bf7bde6ec..b0f1427d6 100644 --- a/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java +++ b/commonmark/src/test/java/org/commonmark/test/LinkReferenceDefinitionNodeTest.java @@ -17,9 +17,11 @@ public void testDefinitionWithoutParagraph() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url 'title'"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(2)); + // Node size is 3 instead of 2 due to BlankLine pseudo-node + assertThat(nodes.size(), is(3)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - LinkReferenceDefinition definition = assertDef(nodes.get(1), "foo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + LinkReferenceDefinition definition = assertDef(nodes.get(2), "foo"); assertThat(definition.getDestination(), is("/url")); assertThat(definition.getTitle(), is("title")); @@ -41,10 +43,12 @@ public void testMultipleDefinitions() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url\n[bar]: /url"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(3)); + // Node size is 4 instead of 3 due to BlankLine pseudo-node + assertThat(nodes.size(), is(4)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - assertDef(nodes.get(1), "foo"); - assertDef(nodes.get(2), "bar"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + assertDef(nodes.get(2), "foo"); + assertDef(nodes.get(3), "bar"); } @Test @@ -52,13 +56,15 @@ public void testMultipleDefinitionsWithSameLabel() { Node document = parse("This is a paragraph with a [foo] link.\n\n[foo]: /url1\n[foo]: /url2"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(3)); + // Node size is 4 instead of 3 due to BlankLine pseudo-node + assertThat(nodes.size(), is(4)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - LinkReferenceDefinition def1 = assertDef(nodes.get(1), "foo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + LinkReferenceDefinition def1 = assertDef(nodes.get(2), "foo"); assertThat(def1.getDestination(), is("/url1")); // When there's multiple definitions with the same label, the first one "wins", as in reference links will use // that. But we still want to preserve the original definitions in the document. - LinkReferenceDefinition def2 = assertDef(nodes.get(2), "foo"); + LinkReferenceDefinition def2 = assertDef(nodes.get(3), "foo"); assertThat(def2.getDestination(), is("/url2")); } @@ -110,9 +116,11 @@ public void testDefinitionLabelCaseIsPreserved() { Node document = parse("This is a paragraph with a [foo] link.\n\n[fOo]: /url 'title'"); List nodes = Nodes.getChildren(document); - assertThat(nodes.size(), is(2)); + // Node size is 3 instead of 2 due to BlankLine pseudo-node + assertThat(nodes.size(), is(3)); assertThat(nodes.get(0), instanceOf(Paragraph.class)); - assertDef(nodes.get(1), "fOo"); + assertThat(nodes.get(1), instanceOf(BlankLine.class)); + assertDef(nodes.get(2), "fOo"); } private static Node parse(String input) { diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index e486750aa..0dd759746 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -155,6 +155,31 @@ private String firstText(Node n) { } private static class DashBlock extends CustomBlock { + + @Override + public String whitespacePreBlock() { + return ""; + } + + @Override + public String whitespacePreContent() { + return ""; + } + + @Override + public String whitespacePostContent() { + return ""; + } + + @Override + public String whitespacePostBlock() { + return ""; + } + + @Override + public void setWhitespace(String... newWhitespace) { + // No whitespace needed for testing + } } private static class DashBlockParser extends AbstractBlockParser { From 23fd13993b6e53f08279f36d4d63697032c21472 Mon Sep 17 00:00:00 2001 From: Seii Date: Wed, 25 Aug 2021 11:58:40 -0600 Subject: [PATCH 2/5] Fix link regression --- .../main/java/org/commonmark/internal/InlineParserImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index e43bba88b..fd7216596 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -364,12 +364,14 @@ private Node parseCloseBracket(String prefix) { whitespacePreDestination = ""; } else { whitespacePreTitle = scanner.whitespaceAsString(); - scanner.whitespace(); + // title needs a whitespace before if (whitespacePreTitle.length() >= 1) { + Position titleStart = scanner.position(); // Capture the symbol used for the title titleSymbol = scanner.peek(); rawTitle = parseLinkTitleRaw(scanner); + scanner.setPosition(titleStart); title = parseLinkTitle(scanner); // If title isn't valid, discard the captured value of its symbol @@ -585,8 +587,6 @@ private String parseLinkTitleRaw(Scanner rawScanner) { String rawTitle = scanner.getSource(start, scanner.position()).getContent(); - rawScanner.setPosition(start); - return rawTitle; } From db6c947fd3c907a63a87ac469c9db4efa4ff2a8c Mon Sep 17 00:00:00 2001 From: Seii Date: Sun, 29 Aug 2021 15:11:23 -0600 Subject: [PATCH 3/5] Replace Java 8 incompabilities --- .../java/org/commonmark/internal/BlockQuoteParser.java | 2 +- .../org/commonmark/internal/IndentedCodeBlockParser.java | 2 +- .../java/org/commonmark/internal/InlineParserImpl.java | 6 +++--- .../java/org/commonmark/internal/ListBlockParser.java | 8 ++++---- .../java/org/commonmark/internal/ParagraphParser.java | 4 ++-- .../internal/inline/EmphasisDelimiterProcessor.java | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java index 87f016689..ea8729786 100644 --- a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java @@ -59,7 +59,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar String preBlockWhitespace = ""; if(nextNonSpace > 0) { - if(state.getLine().getContent().subSequence(0, nextNonSpace).toString().isBlank()) { + if(state.getLine().getContent().subSequence(0, nextNonSpace).toString().trim().equals("")) { preBlockWhitespace = state.getLine().getContent().subSequence(0, nextNonSpace).toString(); }else { preBlockWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace, 0); diff --git a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java index d4da4bff8..6649c8e73 100644 --- a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java @@ -59,7 +59,7 @@ public void addLine(SourceLine line) { // Strip leading whitespace off first raw line (because it's already // captured by the whitespace tracker) if(rawLines != null && rawLines.size() == 0) { - rawLines.add(line.getContent().toString().stripLeading()); + rawLines.add(line.getContent().toString().replaceFirst("^\\s+", "")); }else { rawLines.add(prefix + line.getContent()); } diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index fd7216596..6b363995e 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -244,7 +244,7 @@ private List parseInline() { // Preserve a prefix which has _only_ whitespace, as this may be // needed in the following "parseText" method - if(!prefix.isEmpty() && prefix.isBlank()) { + if(!prefix.isEmpty() && prefix.trim().equals("")) { prefix = ""; } @@ -639,7 +639,7 @@ private Node parseText(String prefix) { if(!prefix.isEmpty()) { // Capture whitespace and anything that might be a block quote for roundtrip purposes - if(prefix.isBlank() || prefix.contains(">")) { + if(prefix.trim().equals("") || prefix.contains(">")) { preContentWhitespace = prefix; } } @@ -793,7 +793,7 @@ private void processDelimiters(Delimiter stackBottom, String prefix) { usedDelims = delimiterProcessor.process(opener, closer, prefix); String openerWhitespace = opener.characters.get(0).whitespacePreContent(); - if(!openerWhitespace.isBlank()) { + if(!openerWhitespace.trim().equals("")) { closer.characters.get(0).setWhitespace(openerWhitespace, ""); } diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java index f2b438b4e..6ecc83015 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java @@ -142,7 +142,7 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { // AST: Lists can be separated from their delimiter by a blank line, make // sure to capture this for roundtrip purposes if it occurs - if(line.subSequence(index + 1, line.length()).toString().isBlank()) { + if(line.subSequence(index + 1, line.length()).toString().trim().equals("")) { firstLineBlank = true; }else { firstLineBlank = false; @@ -150,7 +150,7 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { String preBlockWhitespace = line.subSequence(0, index).toString(); - if(!preBlockWhitespace.isBlank()) { + if(!preBlockWhitespace.trim().equals("")) { preBlockWhitespace = ""; } @@ -196,7 +196,7 @@ private static ListMarkerData parseOrderedList(CharSequence line, int index) { // AST: Lists can be separated from their delimiter by a blank line, make // sure to capture this for roundtrip purposes if it occurs - if(line.subSequence(index + 1, line.length()).toString().isBlank()) { + if(line.subSequence(index + 1, line.length()).toString().trim().equals("")) { firstLineBlank = true; }else { firstLineBlank = false; @@ -207,7 +207,7 @@ private static ListMarkerData parseOrderedList(CharSequence line, int index) { String preContentWhitespace = Parsing.collectWhitespace(line, i+1, line.length()); String preBlockWhitespace = line.subSequence(0, index).toString(); - if(!preBlockWhitespace.isBlank()) { + if(!preBlockWhitespace.trim().equals("")) { preBlockWhitespace = ""; } diff --git a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java index e3bb33a61..4ae4fe4f4 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java @@ -50,8 +50,8 @@ public BlockContinue tryContinue(ParserState state) { if (!state.isBlank()) { - if(state.getLine() != null && state.getLine().getContent().toString().stripLeading().length() != 0) { - setextCheck = state.getLine().getContent().toString().stripLeading().charAt(0); + if(state.getLine() != null && state.getLine().getContent().toString().replaceFirst("^\\s+", "").length() != 0) { + setextCheck = state.getLine().getContent().toString().replaceFirst("^\\s+", "").charAt(0); } if(setextCheck != Character.MIN_VALUE && setextCheck != '=' && setextCheck != '-') { diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java index 1a32b1f5e..47708e8f5 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EmphasisDelimiterProcessor.java @@ -79,7 +79,7 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun, String pref String preContentWhitespace = prefix; if(!openingRun.getOpener().whitespacePreContent().isEmpty() && - openingRun.getOpener().whitespacePreContent().isBlank()) { + openingRun.getOpener().whitespacePreContent().trim().equals("")) { preContentWhitespace = openingRun.getOpener().whitespacePreContent(); } From 7269be28589a48590ad37573d58805b9641ff72a Mon Sep 17 00:00:00 2001 From: Seii Date: Tue, 7 Sep 2021 12:49:39 -0600 Subject: [PATCH 4/5] Code fixes after first PR review --- .../StrikethroughDelimiterProcessor.java | 6 + .../ins/internal/InsDelimiterProcessor.java | 5 + .../commonmark/internal/BlockQuoteParser.java | 21 +- .../commonmark/internal/DocumentParser.java | 59 ++-- .../internal/FencedCodeBlockParser.java | 12 +- .../commonmark/internal/HeadingParser.java | 30 +- .../commonmark/internal/HtmlBlockParser.java | 2 +- .../internal/IndentedCodeBlockParser.java | 10 +- .../commonmark/internal/InlineParserImpl.java | 12 +- .../LinkReferenceDefinitionParser.java | 22 +- .../commonmark/internal/ListBlockParser.java | 8 +- .../commonmark/internal/ListItemParser.java | 13 +- .../commonmark/internal/ParagraphParser.java | 22 +- .../internal/ThematicBreakParser.java | 12 +- .../inline/BackticksInlineParser.java | 2 +- .../internal/inline/BlankLineParser.java | 30 +- .../commonmark/internal/util/LinkScanner.java | 3 - .../org/commonmark/node/AbstractVisitor.java | 4 +- .../java/org/commonmark/node/BlankLine.java | 57 +--- .../main/java/org/commonmark/node/Block.java | 36 --- .../java/org/commonmark/node/BlockQuote.java | 37 +-- .../java/org/commonmark/node/BulletList.java | 31 -- .../java/org/commonmark/node/Document.java | 31 +- .../org/commonmark/node/FencedCodeBlock.java | 49 ++-- .../java/org/commonmark/node/Heading.java | 51 ++-- .../java/org/commonmark/node/HtmlBlock.java | 31 -- .../main/java/org/commonmark/node/Image.java | 2 +- .../commonmark/node/IndentedCodeBlock.java | 36 +-- .../main/java/org/commonmark/node/Link.java | 2 +- .../node/LinkReferenceDefinition.java | 4 +- .../java/org/commonmark/node/ListBlock.java | 19 ++ .../java/org/commonmark/node/ListItem.java | 35 +-- .../main/java/org/commonmark/node/Node.java | 15 - .../main/java/org/commonmark/node/Nodes.java | 3 + .../java/org/commonmark/node/OrderedList.java | 31 -- .../java/org/commonmark/node/Paragraph.java | 30 +- .../main/java/org/commonmark/node/Text.java | 2 +- .../org/commonmark/node/ThematicBreak.java | 31 -- .../java/org/commonmark/node/Visitor.java | 4 +- .../org/commonmark/parser/SourceLines.java | 3 +- .../commonmark/parser/block/BlockParser.java | 5 +- .../parser/delimiter/DelimiterProcessor.java | 1 + .../parser/delimiter/DelimiterRun.java | 4 + .../renderer/roundtrip/CommonMarkWriter.java | 2 +- .../roundtrip/CoreCommonMarkNodeRenderer.java | 270 +++++++++--------- .../java/org/commonmark/test/ParserTest.java | 25 -- 46 files changed, 404 insertions(+), 716 deletions(-) diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java index 7d54eedf2..de2073487 100644 --- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java +++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java @@ -53,4 +53,10 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 0; } } + + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java index b0bfb4c6e..b112f510f 100644 --- a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java +++ b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsDelimiterProcessor.java @@ -53,4 +53,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 0; } } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java index ea8729786..901d6a04e 100644 --- a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java @@ -11,11 +11,12 @@ public class BlockQuoteParser extends AbstractBlockParser { // Preserve original default constructor by explicitly defining one public BlockQuoteParser() { - super(); + super(); } - public BlockQuoteParser(String preBlockWhitespace, String postMarkerWhitespace) { - block.setWhitespace(preBlockWhitespace, postMarkerWhitespace); + public BlockQuoteParser(String preMarkerWhitespace, String postMarkerWhitespace) { + block.setPreMarkerWhitespace(preMarkerWhitespace); + block.setPostMarkerWhitespace(postMarkerWhitespace); } @Override @@ -56,28 +57,24 @@ private static boolean isMarker(ParserState state, int index) { public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { int nextNonSpace = state.getNextNonSpaceIndex(); - String preBlockWhitespace = ""; + String preMarkerWhitespace = ""; if(nextNonSpace > 0) { - if(state.getLine().getContent().subSequence(0, nextNonSpace).toString().trim().equals("")) { - preBlockWhitespace = state.getLine().getContent().subSequence(0, nextNonSpace).toString(); - }else { - preBlockWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace, 0); - } + preMarkerWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), nextNonSpace - 1, 0); } if (isMarker(state, nextNonSpace)) { int newColumn = state.getColumn() + state.getIndent() + 1; // optional following space or tab - String optionalWhitespace = ""; + String postMarkerWhitespace = ""; if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) { - optionalWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), nextNonSpace + 1, state.getLine().getContent().length() - 1); + postMarkerWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), nextNonSpace + 1, state.getLine().getContent().length() - 1); newColumn++; } - return BlockStart.of(new BlockQuoteParser(preBlockWhitespace, optionalWhitespace)).atColumn(newColumn); + return BlockStart.of(new BlockQuoteParser(preMarkerWhitespace, postMarkerWhitespace)).atColumn(newColumn); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java index 2506779cd..210843fca 100644 --- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java @@ -309,13 +309,13 @@ private void parseLine(CharSequence ln) { } if (!blockParser.isContainer()) { - // Capture any blank lines within a separate node for roundtrip purposes - // BlankLine nodes do _not_ interrupt the current node type's processor - if(isBlank()) { - BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); - addChild(new OpenBlockParser(blankLineParser, lastIndex)); - } - + // Capture any blank lines within a separate node for roundtrip purposes + // BlankLine nodes do _not_ interrupt the current node type's processor + if(isBlank()) { + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } + addLine(); } else if (!isBlank()) { // create paragraph container for line @@ -335,9 +335,9 @@ private void parseLine(CharSequence ln) { // Capture any blank lines within a separate node for roundtrip purposes // BlankLine nodes do _not_ interrupt the current node type's processor if(isBlank()) { - BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); - addChild(new OpenBlockParser(blankLineParser, lastIndex)); - } + BlankLineParser blankLineParser = new BlankLineParser(ln.toString()); + addChild(new OpenBlockParser(blankLineParser, lastIndex)); + } } } } @@ -445,20 +445,20 @@ private void addLine() { sb.append(rest); content = sb.toString(); } else { - content = line.getContent().toString(); + content = line.getContent().toString(); } SourceSpan sourceSpan = null; if (includeSourceSpans == IncludeSourceSpans.BLOCKS_AND_INLINES) { // Note that if we're in a partially-consumed tab, the length here corresponds to the content but not to the // actual source length. That sounds like a problem, but I haven't found a test case where it matters (yet). - sourceSpan = SourceSpan.of(lineIndex, index, line.getLiteralLine().getContent().length()); + sourceSpan = SourceSpan.of(lineIndex, index, line.getLiteralLine().getContent().length()); } if(!columnIsInTab) { - getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, index)); + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, index)); }else { - getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, 0)); + getActiveBlockParser().addLine(SourceLine.of(content, sourceSpan, 0)); } addSourceSpans(); @@ -473,7 +473,7 @@ private void addSourceSpans() { int length = line.getContent().length() - blockIndex; if(line.getLiteralIndex() != 0) { - length = line.getLiteralLine().getContent().length() - blockIndex; + length = line.getLiteralLine().getContent().length() - blockIndex; } if (length != 0) { @@ -532,20 +532,20 @@ private void processInlines() { * its parent, and so on until we find a block that can accept children. */ private void addChild(OpenBlockParser openBlockParser) { - // BlankLine nodes do _not_ interrupt the current node type's processor, as - // they are non-standard CommonMark nodes meant only for roundtrip processing - if(!(openBlockParser.blockParser instanceof BlankLineParser)) { - while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { - closeBlockParsers(1); - } - } + // BlankLine nodes do _not_ interrupt the current node type's processor, as + // they are non-standard CommonMark nodes meant only for roundtrip processing + if(!(openBlockParser.blockParser instanceof BlankLineParser)) { + while (!getActiveBlockParser().canContain(openBlockParser.blockParser.getBlock())) { + closeBlockParsers(1); + } + } getActiveBlockParser().getBlock().appendChild(openBlockParser.blockParser.getBlock()); // BlankLine nodes do _not_ become the active parser, as this would interrupt the // parser before it if(!(openBlockParser.blockParser instanceof BlankLineParser)) { - activateBlockParser(openBlockParser); + activateBlockParser(openBlockParser); } } @@ -581,16 +581,15 @@ private Document finalizeAndProcess() { closeBlockParsers(openBlockParsers.size()); processInlines(); Document docNode = documentBlockParser.getBlock(); - String postBlockWhitespace = ""; + String whitespaceEndOfDocument = ""; // CommonMark test cases end with newlines, indicating that this should be standard - // in rountrip scenarios. If a post-document whitespace doesn't exist, add it. - if(postBlockWhitespace.isEmpty()) { - postBlockWhitespace = "\n"; - } + // in roundtrip scenarios. If a post-document whitespace doesn't exist, add it. + if(whitespaceEndOfDocument.isEmpty()) { + whitespaceEndOfDocument = "\n"; + } - docNode.setWhitespace(docNode.whitespacePreBlock(), docNode.whitespacePreContent(), - docNode.whitespacePostContent(), postBlockWhitespace); + docNode.setEndOfDocumentWhitespace(whitespaceEndOfDocument); return documentBlockParser.getBlock(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java index 074c6d9b1..9f9982592 100644 --- a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java @@ -25,7 +25,7 @@ public FencedCodeBlockParser(char fenceChar, int fenceLength, int fenceIndent) { // Fenced code blocks can't be indented by newlines or tabs, so it's safe to assume // spaces for the indentation - block.setWhitespace(Parsing.generateSpaces(fenceIndent)); + block.setPreStartFenceWhitespace(Parsing.generateSpaces(fenceIndent)); } @Override @@ -39,15 +39,15 @@ public BlockContinue tryContinue(ParserState state) { int newIndex = state.getIndex(); CharSequence line = state.getLine().getContent(); if (state.getIndent() < Parsing.CODE_BLOCK_INDENT && nextNonSpace < line.length() && line.charAt(nextNonSpace) == block.getFenceChar() && isClosing(line, nextNonSpace)) { - // Capture whitespace before closing fence (if any) for roundtrip purposes + // Capture whitespace before closing fence (if any) for roundtrip purposes if(nextNonSpace > 0) { - block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), Parsing.collectWhitespace(line, 0, nextNonSpace), block.whitespacePostBlock()); + block.setPreEndFenceWhitespace(Parsing.collectWhitespace(line, 0, nextNonSpace)); } - // closing fence - we're at end of line, so we can finalize now + // closing fence - we're at end of line, so we can finalize now return BlockContinue.finished(); } else { - // Capture line before optional spaces are removed + // Capture line before optional spaces are removed rawLines.append(line.toString()); rawLines.append('\n'); @@ -183,7 +183,7 @@ private boolean isClosing(CharSequence line, int index) { // Capture post-fence spaces (if any) for roundtrip purposes if(after > 0 && after != (index + fences)) { - block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), block.whitespacePostContent(), Parsing.collectWhitespaceBackwards(line, line.length() - 1, 0)); + block.setPostBlockWhitespace(Parsing.collectWhitespaceBackwards(line, line.length() - 1, 0)); } return after == line.length(); diff --git a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java index 702300db5..fd3b642bc 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java @@ -21,27 +21,19 @@ public class HeadingParser extends AbstractBlockParser { private final SourceLines content; public HeadingParser(int level, SourceLines content) { - block.setLevel(level); + block.setLevel(level); block.setSymbolType('#'); this.content = content; } - public HeadingParser(int level, SourceLines content, char symbolType, int numEndingSymbols, String... whitespace) { + public HeadingParser(int level, SourceLines content, char symbolType, int numEndingSymbols, String whitespacePreBlock, String whitespacePreContent, String whitespacePostContent, String whitespacePostBlock) { this(level, content); block.setSymbolType(symbolType); block.setNumEndingSymbol(numEndingSymbols); - - if(whitespace.length == 4) { - block.setWhitespace(whitespace); - }else { - String[] tempArray = {"", "", "", ""}; - - for(int i = 0; i < whitespace.length; i++) { - tempArray[i] = whitespace[i]; - } - - block.setWhitespace(tempArray); - } + block.setPreBlockWhitespace(whitespacePreBlock); + block.setPreContentWhitespace(whitespacePreContent); + block.setPostContentWhitespace(whitespacePostContent); + block.setPostBlockWhitespace(whitespacePostBlock); } @Override @@ -97,12 +89,8 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar if (setextHeadingLevel > 0) { int numEndingSymbols = line.getContent().toString().trim().length(); - // AST: Setext headings have a slightly different twist on whitespace: - // Pre-block = Before setext text begins - // Pre-content = Directly after setext text - // Post-content = Before setext delimiting line - // Post-block = After setext delimiting line - String preContentWhitespace = ""; + // AST: Setext headings have a slightly different twist on + // whitespace, see the 

Heading
class for details String postContentWhitespace = Parsing.collectWhitespace(line.getContent(), 0, line.getContent().length()); String postBlockWhitespace = Parsing.collectWhitespaceBackwards(line.getContent(), line.getContent().length() - 1, 0); @@ -134,7 +122,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar SourceLines paragraph = matchedBlockParser.getParagraphLines(); if (!paragraph.isEmpty()) { - return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph, symbolType, numEndingSymbols, preBlockWhitespace, preContentWhitespace, postContentWhitespace, postBlockWhitespace)) + return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph, symbolType, numEndingSymbols, preBlockWhitespace, "", postContentWhitespace, postBlockWhitespace)) .atIndex(line.getContent().length()) .replaceActiveBlockParser(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java index b240681f2..2dc964217 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java @@ -96,7 +96,7 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - // Raw line and literal line are the same + // Raw line and literal line are the same if(line.getLiteralIndex() == 0) { content.add(line.getContent()); rawContent.add(line.getContent()); diff --git a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java index 6649c8e73..65f34e665 100644 --- a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java @@ -26,8 +26,8 @@ public IndentedCodeBlockParser() { super(); } - public IndentedCodeBlockParser(String preBlockWhitespace, String preContentWhitespace) { - block.setWhitespace(preBlockWhitespace, preContentWhitespace); + public IndentedCodeBlockParser(String indentWhitespace) { + block.setIndentWhitespace(indentWhitespace); } @Override @@ -48,7 +48,7 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - String prefix = ""; + String prefix = ""; // Capture the prefix (needed for roundtrip, but unnecessary for HTML processing) if(line.getLiteralIndex() != 0) { @@ -113,9 +113,9 @@ public static class Factory extends AbstractBlockParserFactory { public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { // An indented code block cannot interrupt a paragraph. if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT && !state.isBlank() && !(state.getActiveBlockParser().getBlock() instanceof Paragraph)) { - String preContentWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()); + String indentWhitespace = Parsing.collectWhitespace(state.getLine().getContent(), 0, state.getLine().getContent().length()); - return BlockStart.of(new IndentedCodeBlockParser("", preContentWhitespace)).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); + return BlockStart.of(new IndentedCodeBlockParser(indentWhitespace)).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index 6b363995e..c9904f9d3 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -184,7 +184,7 @@ private Text text(SourceLines sourceLines, String preContentWhitespace, String p * On failure, return null. */ private List parseInline() { - // AST: Capture raw information needed for roundtrip rendering + // AST: Capture raw information needed for roundtrip rendering String prefix = scanner.alignToLiteral(); char c = scanner.peek(); @@ -206,7 +206,7 @@ private List parseInline() { // No inline parser, delimiter or other special handling. if (!specialCharacters.get(c)) { - List nodeList = Collections.singletonList(parseText(prefix)); + List nodeList = Collections.singletonList(parseText(prefix)); prefix = ""; return nodeList; } @@ -215,14 +215,14 @@ private List parseInline() { if (inlineParsers != null) { Position position = scanner.position(); for (InlineContentParser inlineParser : inlineParsers) { - ParsedInline parsedInline; - if(inlineParser instanceof BackticksInlineParser) { + ParsedInline parsedInline; + if(inlineParser instanceof BackticksInlineParser) { parsedInline = inlineParser.tryParse(this, prefix); prefix = ""; }else { parsedInline = inlineParser.tryParse(this); } - + if (parsedInline instanceof ParsedInlineImpl) { ParsedInlineImpl parsedInlineImpl = (ParsedInlineImpl) parsedInline; Node node = parsedInlineImpl.getNode(); @@ -931,7 +931,7 @@ private void mergeTextNodesInclusive(Node fromNode, Node toNode) { } private void mergeIfNeeded(Text first, Text last, int textLength) { - if (first != null && last != null && first != last) { + if (first != null && last != null && first != last) { StringBuilder sb = new StringBuilder(textLength); StringBuilder sb2 = new StringBuilder(textLength); sb.append(first.getLiteral()); diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java index 96790a797..866c6f4e7 100644 --- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java @@ -41,10 +41,10 @@ public class LinkReferenceDefinitionParser { private String whitespacePostTitle = ""; public void parse(SourceLine line) { - paragraphLines.add(line.getLiteralLine()); + paragraphLines.add(line.getLiteralLine()); rawParagraphLines.add(line); - if (state == State.PARAGRAPH) { + if (state == State.PARAGRAPH) { // We're in a paragraph now. Link reference definitions can only appear at the beginning, so once // we're in a paragraph, there's no going back. return; @@ -65,7 +65,7 @@ public void parse(SourceLine line) { break; } case LABEL: { - // Capture line number to determine if line breaks occur + // Capture line number to determine if line breaks occur if(line.getSourceSpan() != null) { previousLineIndex = line.getSourceSpan().getLineIndex(); } @@ -74,7 +74,7 @@ public void parse(SourceLine line) { break; } case DESTINATION: { - // Check for any line breaks + // Check for any line breaks if(line.getSourceSpan() != null) { if(line.getSourceSpan().getLineIndex() > previousLineIndex) { whitespacePreDestination = whitespacePreDestination + "\n"; @@ -86,7 +86,7 @@ public void parse(SourceLine line) { break; } case START_TITLE: { - // Check for any line breaks + // Check for any line breaks if(line.getSourceSpan() != null) { if(line.getSourceSpan().getLineIndex() > previousLineIndex) { whitespacePreTitle = whitespacePreTitle + "\n"; @@ -150,7 +150,7 @@ State getState() { } private boolean startDefinition(Scanner scanner) { - // Capture whitespace for roundtrip purposes + // Capture whitespace for roundtrip purposes whitespacePreLabel = scanner.whitespaceAsString(); String whitespaceCheck = scanner.alignToLiteral(); @@ -159,7 +159,7 @@ private boolean startDefinition(Scanner scanner) { whitespacePreLabel = whitespaceCheck; } - if (!scanner.next('[')) { + if (!scanner.next('[')) { return false; } @@ -210,7 +210,7 @@ private boolean label(Scanner scanner) { } private boolean destination(Scanner scanner) { - whitespacePreDestination = whitespacePreDestination + + whitespacePreDestination = whitespacePreDestination + scanner.whitespaceAsString(); Position start = scanner.position(); if (!LinkScanner.scanLinkDestination(scanner)) { @@ -239,8 +239,8 @@ private boolean destination(Scanner scanner) { } private boolean startTitle(Scanner scanner) { - whitespacePreTitle = whitespacePreTitle + scanner.whitespaceAsString(); - if (!scanner.hasNext()) { + whitespacePreTitle = whitespacePreTitle + scanner.whitespaceAsString(); + if (!scanner.hasNext()) { state = State.START_DEFINITION; return true; } @@ -291,7 +291,7 @@ private boolean title(Scanner scanner) { scanner.next(); whitespacePostTitle = scanner.whitespaceAsString(); if (scanner.hasNext()) { - title = null; + title = null; whitespacePostTitle = ""; // spec: No further non-whitespace characters may occur on the line. return false; diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java index 6ecc83015..202434ceb 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java @@ -135,7 +135,7 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { case '-': case '+': case '*': - if (isSpaceTabOrEnd(line, index + 1)) { + if (isSpaceTabOrEnd(line, index + 1)) { // Collect any pre-content whitespace in the first line for // roundtrip purposes. String preContentWhitespace = Parsing.collectWhitespace(line, index + 1, line.length()); @@ -156,7 +156,8 @@ private static ListMarkerData parseListMarker(CharSequence line, int index) { BulletList bulletList = new BulletList(); bulletList.setBulletMarker(c); - bulletList.setWhitespace(preBlockWhitespace, preContentWhitespace); + bulletList.setPreBlockWhitespace(preBlockWhitespace); + bulletList.setPreContentWhitespace(preContentWhitespace); return new ListMarkerData(bulletList, index + 1); } else { return null; @@ -217,7 +218,8 @@ private static ListMarkerData parseOrderedList(CharSequence line, int index) { // only auto-incremented during rendering currentRawNumber = number; orderedList.setDelimiter(c); - orderedList.setWhitespace(preBlockWhitespace, preContentWhitespace); + orderedList.setPreBlockWhitespace(preBlockWhitespace); + orderedList.setPreContentWhitespace(preContentWhitespace); return new ListMarkerData(orderedList, i + 1); } else { return null; diff --git a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java index 45a80c71f..389031173 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java @@ -29,11 +29,12 @@ public ListItemParser(int contentIndent) { block.setRawNumber(""); } - public ListItemParser(int contentIndent, String currentRawNumber, boolean firstLineBlank, String whitespacePreBlock, String whitespacePreContent) { + public ListItemParser(int contentIndent, String currentRawNumber, boolean firstLineBlank, String whitespacePreMarker, String whitespacePostMarker) { this(contentIndent); block.setRawNumber(currentRawNumber); this.firstLineBlank = firstLineBlank; - block.setWhitespace(whitespacePreBlock, whitespacePreContent); + block.setPreMarkerWhitespace(whitespacePreMarker); + block.setPostMarkerWhitespace(whitespacePostMarker); } @Override @@ -64,7 +65,7 @@ public Block getBlock() { @Override public BlockContinue tryContinue(ParserState state) { if (state.isBlank()) { - if (block.getFirstChild() == null || block.getFirstChild() instanceof BlankLine) { + if (block.getFirstChild() == null || block.getFirstChild() instanceof BlankLine) { // Blank line after empty list item return BlockContinue.none(); } else { @@ -77,9 +78,9 @@ public BlockContinue tryContinue(ParserState state) { } if (state.getIndent() >= contentIndent) { - if(firstLineBlank) { - String whitespacePreContent = block.whitespacePreContent() + "\n"; - block.setWhitespace(block.whitespacePreBlock(), whitespacePreContent); + if(firstLineBlank) { + String whitespacePostMarker = block.whitespacePostMarker() + "\n"; + block.setPostMarkerWhitespace(whitespacePostMarker); firstLineBlank = false; } return BlockContinue.atColumn(state.getColumn() + contentIndent); diff --git a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java index 4ae4fe4f4..7a39a6c40 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java @@ -2,7 +2,6 @@ import java.util.List; -import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.LinkReferenceDefinition; import org.commonmark.node.Paragraph; @@ -19,18 +18,13 @@ public class ParagraphParser extends AbstractBlockParser { private final Paragraph block = new Paragraph(); private final LinkReferenceDefinitionParser linkReferenceDefinitionParser = new LinkReferenceDefinitionParser(); - private String postContentWhitespace = ""; - private String postBlockWhitespace = ""; - // Preserve original default constructor by explicitly defining one public ParagraphParser() { super(); } public ParagraphParser(String preBlockWhitespace) { - block.setWhitespace(preBlockWhitespace); - postContentWhitespace = ""; - postBlockWhitespace = ""; + block.setPreBlockWhitespace(preBlockWhitespace); } @Override @@ -45,19 +39,7 @@ public Block getBlock() { @Override public BlockContinue tryContinue(ParserState state) { - char setextCheck = Character.MIN_VALUE; - postBlockWhitespace = block.whitespacePostBlock(); - if (!state.isBlank()) { - - if(state.getLine() != null && state.getLine().getContent().toString().replaceFirst("^\\s+", "").length() != 0) { - setextCheck = state.getLine().getContent().toString().replaceFirst("^\\s+", "").charAt(0); - } - - if(setextCheck != Character.MIN_VALUE && setextCheck != '=' && setextCheck != '-') { - // Collect post-content whitespace for roundtrip purposes - postContentWhitespace = Parsing.collectWhitespaceBackwards(state.getLine().getContent(), state.getLine().getContent().length() - 1, 0); - } return BlockContinue.atIndex(state.getIndex()); } else { return BlockContinue.none(); @@ -78,8 +60,6 @@ public void addSourceSpan(SourceSpan sourceSpan) { @Override public void closeBlock() { - block.setWhitespace(block.whitespacePreBlock(), block.whitespacePreContent(), postContentWhitespace, postBlockWhitespace); - if (linkReferenceDefinitionParser.getParagraphLines().isEmpty()) { block.unlink(); } else { diff --git a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java index 78192e196..f1c21e83f 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ThematicBreakParser.java @@ -1,6 +1,5 @@ package org.commonmark.internal; -import org.commonmark.internal.util.Parsing; import org.commonmark.node.Block; import org.commonmark.node.ThematicBreak; import org.commonmark.parser.block.AbstractBlockParser; @@ -23,11 +22,6 @@ public ThematicBreakParser(CharSequence content) { block.setContent(content); } - public ThematicBreakParser(CharSequence content, String... whitespace) { - block.setContent(content); - block.setWhitespace(whitespace); - } - @Override public Block getBlock() { return block; @@ -49,12 +43,8 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar int nextNonSpace = state.getNextNonSpaceIndex(); CharSequence line = state.getLine().getContent(); - // Collect pre- and post-content whitespace for roundtrip purposes - String preContentWhitespace = Parsing.collectWhitespace(line, 0, nextNonSpace); - String postContentWhitespace = Parsing.collectWhitespaceBackwards(line, line.length() - 1, nextNonSpace); - if (isThematicBreak(line, nextNonSpace)) { - return BlockStart.of(new ThematicBreakParser(line, "", preContentWhitespace, postContentWhitespace)).atIndex(line.length()); + return BlockStart.of(new ThematicBreakParser(line)).atIndex(line.length()); } else { return BlockStart.none(); } diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java index 64a1a178c..9da813048 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java @@ -30,7 +30,7 @@ public ParsedInline tryParse(InlineParserState inlineParserState) { if (count == openingTicks) { Code node = new Code(); -node.setNumBackticks(count); + node.setNumBackticks(count); String content = scanner.getSource(afterOpening, beforeClosing).getContent(); // Capture raw string for roundtrip rendering diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java index 8b59bf574..e50e00c11 100644 --- a/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BlankLineParser.java @@ -7,22 +7,22 @@ import org.commonmark.parser.block.ParserState; public class BlankLineParser extends AbstractBlockParser { - - private BlankLine block; - - public BlankLineParser(String raw) { - block = new BlankLine(raw); - } - @Override - public Block getBlock() { - return block; - } + private BlankLine block; - @Override - public BlockContinue tryContinue(ParserState parserState) { - // Blank lines do not become active parsers, so they cannot be continued - return null; - } + public BlankLineParser(String raw) { + block = new BlankLine(raw); + } + + @Override + public Block getBlock() { + return block; + } + + @Override + public BlockContinue tryContinue(ParserState parserState) { + // Blank lines do not become active parsers, so they cannot be continued + return null; + } } \ No newline at end of file diff --git a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java index f492e9f1c..c4513855d 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java @@ -86,9 +86,6 @@ public static boolean scanLinkTitle(Scanner scanner) { return false; } -// if(c != '(' && !(scanner.hasNext() && scanner.peek() == '(')) { -// scanner.next(); -// } scanner.next(); if (!scanLinkTitleContent(scanner, endDelimiter)) { diff --git a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java index 941a6d45e..2d12e3a7c 100644 --- a/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java +++ b/commonmark/src/main/java/org/commonmark/node/AbstractVisitor.java @@ -8,11 +8,11 @@ */ public abstract class AbstractVisitor implements Visitor { - @Override + @Override public void visit(BlankLine blankLine) { visitChildren(blankLine); } - + @Override public void visit(BlockQuote blockQuote) { visitChildren(blockQuote); diff --git a/commonmark/src/main/java/org/commonmark/node/BlankLine.java b/commonmark/src/main/java/org/commonmark/node/BlankLine.java index 4f63c1686..c60cc814d 100644 --- a/commonmark/src/main/java/org/commonmark/node/BlankLine.java +++ b/commonmark/src/main/java/org/commonmark/node/BlankLine.java @@ -2,51 +2,22 @@ public class BlankLine extends Block { - private String raw; - - public BlankLine(String rawContent) { - raw = rawContent; - } + private String raw; - public String getRaw() { - return raw; - } + public BlankLine(String rawContent) { + raw = rawContent; + } - public void setRaw(String raw) { - this.raw = raw; - } + public String getRaw() { + return raw; + } - @Override - public String whitespacePreBlock() { - // Blank lines contain any whitespace as part of their raw content - return ""; - } + public void setRaw(String raw) { + this.raw = raw; + } - @Override - public String whitespacePreContent() { - // Blank lines contain any whitespace as part of their raw content - return ""; - } - - @Override - public String whitespacePostContent() { - // Blank lines contain any whitespace as part of their raw content - return ""; - } - - @Override - public String whitespacePostBlock() { - // Blank lines contain any whitespace as part of their raw content - return ""; - } - - @Override - public void setWhitespace(String... newWhitespace) { - // Blank lines contain any whitespace as part of their raw content - } - - @Override - public void accept(Visitor visitor) { - visitor.visit(this); - } + @Override + public void accept(Visitor visitor) { + visitor.visit(this); + } } diff --git a/commonmark/src/main/java/org/commonmark/node/Block.java b/commonmark/src/main/java/org/commonmark/node/Block.java index 976245d96..753447c5c 100644 --- a/commonmark/src/main/java/org/commonmark/node/Block.java +++ b/commonmark/src/main/java/org/commonmark/node/Block.java @@ -16,40 +16,4 @@ protected void setParent(Node parent) { } super.setParent(parent); } - - // Roundtrip rendering requires capturing various pieces before and after blocks and their contents - public abstract String whitespacePreBlock(); - public abstract String whitespacePreContent(); - public abstract String whitespacePostContent(); - public abstract String whitespacePostBlock(); - public abstract void setWhitespace(String...newWhitespace); - - /** - * All block elements can be impacted structurally by whitespace in some way or another. - * For convenience, this method allows passing any number of strings. However, these strings - * are interpreted as linearly replacing one of 4 pre-defined positions (i.e - * 2 values replace [0] and [1], 3 replace [0], [1], and [2] etc.). These 4 - * values correspond to: - * 0. Pre-block element whitespace - * 1. Pre-content whitespace - * 2. Post-content whitespace - * 3. Post-block element whitespace - * - * If a caller passes more than 4 values, all values past the fourth will be ignored. - * @param newWhitespace - * @return Prepared array of structural whitespace - */ - protected String[] prepareStructuralWhitespace(String... newWhitespace) { - String[] container = {"", "", "", ""}; - - if(newWhitespace.length == 4) { - container = newWhitespace; - }else { - for(int i = 0; i < newWhitespace.length; i++) { - container[i] = newWhitespace[i]; - } - } - - return container; - } } diff --git a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java index bed8119c0..707ca3ec3 100644 --- a/commonmark/src/main/java/org/commonmark/node/BlockQuote.java +++ b/commonmark/src/main/java/org/commonmark/node/BlockQuote.java @@ -1,39 +1,28 @@ package org.commonmark.node; public class BlockQuote extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - + // Whitespace for roundtrip rendering + private String whitespacePreMarker = ""; + private String whitespacePostMarker = ""; + @Override public void accept(Visitor visitor) { visitor.visit(this); } - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; + public String whitespacePreMarker() { + return whitespacePreMarker; } - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; + public String whitespacePostMarker() { + return whitespacePostMarker; } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; + + public void setPreMarkerWhitespace(String whitespace) { + whitespacePreMarker = whitespace; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setPostMarkerWhitespace(String whitespace) { + whitespacePostMarker = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/BulletList.java b/commonmark/src/main/java/org/commonmark/node/BulletList.java index 816a61096..b0db34bba 100644 --- a/commonmark/src/main/java/org/commonmark/node/BulletList.java +++ b/commonmark/src/main/java/org/commonmark/node/BulletList.java @@ -1,13 +1,6 @@ package org.commonmark.node; public class BulletList extends ListBlock { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - private char bulletMarker; @Override @@ -22,28 +15,4 @@ public char getBulletMarker() { public void setBulletMarker(char bulletMarker) { this.bulletMarker = bulletMarker; } - - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; - } - - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); - } } diff --git a/commonmark/src/main/java/org/commonmark/node/Document.java b/commonmark/src/main/java/org/commonmark/node/Document.java index 6c27b6f2d..cfa18c488 100644 --- a/commonmark/src/main/java/org/commonmark/node/Document.java +++ b/commonmark/src/main/java/org/commonmark/node/Document.java @@ -1,39 +1,18 @@ package org.commonmark.node; public class Document extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; + private String whitespaceEndOfDocument = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; + public String whitespaceEndOfDocument() { + return whitespaceEndOfDocument; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setEndOfDocumentWhitespace(String whitespace) { + whitespaceEndOfDocument = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java index 2c55570f3..b56d8952c 100644 --- a/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/FencedCodeBlock.java @@ -1,12 +1,6 @@ package org.commonmark.node; public class FencedCodeBlock extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; private char fenceChar; private int startFenceLength; @@ -15,6 +9,12 @@ public class FencedCodeBlock extends Block { private String info; private String literal; private String raw; + + // Whitespace tracked for roundtrip rendering + private String whitespacePreStartFence = ""; + private String whitespacePreContent = ""; + private String whitespacePreEndFence = ""; + private String whitespacePostBlock = ""; @Override public void accept(Visitor visitor) { @@ -46,15 +46,16 @@ public void setEndFenceLength(int endFenceLength) { } public int getStartFenceIndent() { - return whitespaceTracker[0].length(); + return whitespacePreStartFence.length(); } public int getEndFenceIndent() { - return whitespaceTracker[2].length(); + return whitespacePreEndFence.length(); } /** * @see CommonMark spec + * @return Info string or null */ public String getInfo() { return info; @@ -80,27 +81,35 @@ public void setRaw(String raw) { this.raw = raw; } - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; + public String whitespacePreStartFence() { + return whitespacePreStartFence; } - @Override public String whitespacePreContent() { - return whitespaceTracker[1]; + return whitespacePreContent; } - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; + public String whitespacePreEndFence() { + return whitespacePreEndFence; } - @Override public String whitespacePostBlock() { - return whitespaceTracker[3]; + return whitespacePostBlock; + } + + public void setPreStartFenceWhitespace(String whitespace) { + whitespacePreStartFence = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } + + public void setPreEndFenceWhitespace(String whitespace) { + whitespacePreEndFence = whitespace; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setPostBlockWhitespace(String whitespace) { + whitespacePostBlock = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/Heading.java b/commonmark/src/main/java/org/commonmark/node/Heading.java index 5ffb060b9..a5f37b64d 100644 --- a/commonmark/src/main/java/org/commonmark/node/Heading.java +++ b/commonmark/src/main/java/org/commonmark/node/Heading.java @@ -1,16 +1,19 @@ package org.commonmark.node; public class Heading extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - private int level; private char symbolType; private int numEndingSymbol; + + // Whitespace for roundtrip rendering + // AST: Pre-marker whitespace (ATX) or before setext text (setext) + private String whitespacePreBlock = ""; + // AST: Pre-text (ATX) or after setext text ends (setext) + private String whitespacePreContent = ""; + // AST: Post-text (ATX) or before setext delimiting line (setext) + private String whitespacePostContent = ""; + // AST: Post-block (ATX) or after setext delimiting line (setext) + private String whitespacePostBlock = ""; @Override public void accept(Visitor visitor) { @@ -41,27 +44,35 @@ public void setNumEndingSymbol(int numEndingSymbol) { this.numEndingSymbol = numEndingSymbol; } - @Override public String whitespacePreBlock() { - return whitespaceTracker[0]; + return whitespacePreBlock; } - - @Override + public String whitespacePreContent() { - return whitespaceTracker[1]; + return whitespacePreContent; } - - @Override + public String whitespacePostContent() { - return whitespaceTracker[2]; + return whitespacePostContent; } - - @Override + public String whitespacePostBlock() { - return whitespaceTracker[3]; + return whitespacePostBlock; + } + + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } + + public void setPostContentWhitespace(String whitespace) { + whitespacePostContent = whitespace; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setPostBlockWhitespace(String whitespace) { + whitespacePostBlock = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java index 96ce69f82..6677ecacf 100644 --- a/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/HtmlBlock.java @@ -6,13 +6,6 @@ * @see CommonMark Spec */ public class HtmlBlock extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - private String literal; private String raw; @@ -36,28 +29,4 @@ public String getRaw() { public void setRaw(String raw) { this.raw = raw; } - - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; - } - - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); - } } diff --git a/commonmark/src/main/java/org/commonmark/node/Image.java b/commonmark/src/main/java/org/commonmark/node/Image.java index 74bf313e7..fdc1fcfad 100644 --- a/commonmark/src/main/java/org/commonmark/node/Image.java +++ b/commonmark/src/main/java/org/commonmark/node/Image.java @@ -2,7 +2,7 @@ public class Image extends Node implements LinkFormat { - private String destination; + private String destination; private String rawDestination; private String title; private String rawTitle; diff --git a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java index fb6f599ed..878e0af32 100644 --- a/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/IndentedCodeBlock.java @@ -1,20 +1,16 @@ package org.commonmark.node; public class IndentedCodeBlock extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - - // The "literal" string is optimized for HTML formatting, but omits some details + // The "literal" string is optimized for HTML formatting, but omits some details // which are important for roundtrip rendering private String literal; // The "raw" string is, as much as possible, the entire raw content as it was first // entered into the parser. This allows roundtrip parsing for indented code blocks. private String raw; + + // Actual indent whitespace of a raw string for roundtrip rendering + private String indentWhitespace = ""; @Override public void accept(Visitor visitor) { @@ -37,27 +33,11 @@ public void setRaw(String raw) { this.raw = raw; } - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; + public String getIndentWhitespace() { + return indentWhitespace; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setIndentWhitespace(String whitespace) { + indentWhitespace = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/Link.java b/commonmark/src/main/java/org/commonmark/node/Link.java index 0622453e1..5a7864542 100644 --- a/commonmark/src/main/java/org/commonmark/node/Link.java +++ b/commonmark/src/main/java/org/commonmark/node/Link.java @@ -22,7 +22,7 @@ */ public class Link extends Node implements LinkFormat { - private String destination; + private String destination; private String rawDestination; private String title; private String rawTitle; diff --git a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java index c8e4d3b12..a47d007b3 100644 --- a/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java +++ b/commonmark/src/main/java/org/commonmark/node/LinkReferenceDefinition.java @@ -13,7 +13,7 @@ */ public class LinkReferenceDefinition extends Node { - private String label; + private String label; private String destination; private String rawDestination; private String title; @@ -47,7 +47,7 @@ public String getLabel() { public void setLabel(String label) { this.label = label; } - + public String getDestination() { return destination; } diff --git a/commonmark/src/main/java/org/commonmark/node/ListBlock.java b/commonmark/src/main/java/org/commonmark/node/ListBlock.java index 69482f66e..3f8f129dd 100644 --- a/commonmark/src/main/java/org/commonmark/node/ListBlock.java +++ b/commonmark/src/main/java/org/commonmark/node/ListBlock.java @@ -3,6 +3,10 @@ public abstract class ListBlock extends Block { private boolean tight; + + // Whitespace for roundtrip parsing of the first line in a list block + private String whitespacePreBlock = ""; + private String whitespacePreContent = ""; /** * @return whether this list is tight or loose @@ -16,4 +20,19 @@ public void setTight(boolean tight) { this.tight = tight; } + public String whitespacePreBlock() { + return whitespacePreBlock; + } + + public String whitespacePreContent() { + return whitespacePreContent; + } + + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; + } + + public void setPreContentWhitespace(String whitespace) { + whitespacePreContent = whitespace; + } } diff --git a/commonmark/src/main/java/org/commonmark/node/ListItem.java b/commonmark/src/main/java/org/commonmark/node/ListItem.java index de551b918..0374bdd81 100644 --- a/commonmark/src/main/java/org/commonmark/node/ListItem.java +++ b/commonmark/src/main/java/org/commonmark/node/ListItem.java @@ -1,41 +1,30 @@ package org.commonmark.node; public class ListItem extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; private String rawNumber = ""; + + private String whitespacePreMarker = ""; + private String whitespacePostMarker = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; + public String whitespacePreMarker() { + return whitespacePreMarker; } - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; + public String whitespacePostMarker() { + return whitespacePostMarker; } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; + + public void setPreMarkerWhitespace(String whitespace) { + whitespacePreMarker = whitespace; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setPostMarkerWhitespace(String whitespace) { + whitespacePostMarker = whitespace; } public String getRawNumber() { diff --git a/commonmark/src/main/java/org/commonmark/node/Node.java b/commonmark/src/main/java/org/commonmark/node/Node.java index 39bdc2e5b..d3dff717a 100644 --- a/commonmark/src/main/java/org/commonmark/node/Node.java +++ b/commonmark/src/main/java/org/commonmark/node/Node.java @@ -149,21 +149,6 @@ public void addSourceSpan(SourceSpan sourceSpan) { } this.sourceSpans.add(sourceSpan); } - - /** - * Check whether the given node is of the subset which count as "inline" nodes - * @param node Node to be checked for inline - * @return true if node is an inline node, otherwise false - */ - public static boolean isInlineNode(Node node) { - if(node instanceof Code || node instanceof Emphasis || node instanceof StrongEmphasis || - node instanceof Link || node instanceof Image || node instanceof HtmlInline || - node instanceof HardLineBreak) { - return true; - }else { - return false; - } - } @Override public String toString() { diff --git a/commonmark/src/main/java/org/commonmark/node/Nodes.java b/commonmark/src/main/java/org/commonmark/node/Nodes.java index 22d5932af..da99d63ad 100644 --- a/commonmark/src/main/java/org/commonmark/node/Nodes.java +++ b/commonmark/src/main/java/org/commonmark/node/Nodes.java @@ -14,6 +14,9 @@ private Nodes() { /** * The nodes between (not including) start and end. + * @param start Start node + * @param end End node + * @return Iterable list of 
Node
s */ public static Iterable between(Node start, Node end) { return new NodeIterable(start.getNext(), end); diff --git a/commonmark/src/main/java/org/commonmark/node/OrderedList.java b/commonmark/src/main/java/org/commonmark/node/OrderedList.java index 560453836..d9a1fb51d 100644 --- a/commonmark/src/main/java/org/commonmark/node/OrderedList.java +++ b/commonmark/src/main/java/org/commonmark/node/OrderedList.java @@ -1,13 +1,6 @@ package org.commonmark.node; public class OrderedList extends ListBlock { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - private int startNumber; private String rawNumber; private char delimiter; @@ -40,28 +33,4 @@ public char getDelimiter() { public void setDelimiter(char delimiter) { this.delimiter = delimiter; } - - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; - } - - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); - } } diff --git a/commonmark/src/main/java/org/commonmark/node/Paragraph.java b/commonmark/src/main/java/org/commonmark/node/Paragraph.java index e3e08e308..773b14c45 100644 --- a/commonmark/src/main/java/org/commonmark/node/Paragraph.java +++ b/commonmark/src/main/java/org/commonmark/node/Paragraph.java @@ -4,39 +4,19 @@ * A paragraph block, contains inline nodes such as {@link Text} */ public class Paragraph extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; + // Whitespace for roundtrip rendering + private String whitespacePreBlock = ""; @Override public void accept(Visitor visitor) { visitor.visit(this); } - @Override public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; + return whitespacePreBlock; } - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); + public void setPreBlockWhitespace(String whitespace) { + whitespacePreBlock = whitespace; } } diff --git a/commonmark/src/main/java/org/commonmark/node/Text.java b/commonmark/src/main/java/org/commonmark/node/Text.java index 0b227af91..8edb88093 100644 --- a/commonmark/src/main/java/org/commonmark/node/Text.java +++ b/commonmark/src/main/java/org/commonmark/node/Text.java @@ -4,7 +4,7 @@ public class Text extends Node { - private String literal; + private String literal; private String raw = ""; private String preContentWhitespace; private String postContentWhitespace; diff --git a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java index 5be26fef0..bcd66072d 100644 --- a/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java +++ b/commonmark/src/main/java/org/commonmark/node/ThematicBreak.java @@ -1,13 +1,6 @@ package org.commonmark.node; public class ThematicBreak extends Block { - // Track whitespace as follows: - // [0] Pre-block - // [1] Pre-content - // [2] Post-content - // [3] Post-block - private String[] whitespaceTracker = {"", "", "", ""}; - private CharSequence content; @Override @@ -22,28 +15,4 @@ public CharSequence getContent() { public void setContent(CharSequence content) { this.content = content; } - - @Override - public String whitespacePreBlock() { - return whitespaceTracker[0]; - } - - @Override - public String whitespacePreContent() { - return whitespaceTracker[1]; - } - - @Override - public String whitespacePostContent() { - return whitespaceTracker[2]; - } - - @Override - public String whitespacePostBlock() { - return whitespaceTracker[3]; - } - - public void setWhitespace(String... newWhitespace) { - whitespaceTracker = super.prepareStructuralWhitespace(newWhitespace); - } } diff --git a/commonmark/src/main/java/org/commonmark/node/Visitor.java b/commonmark/src/main/java/org/commonmark/node/Visitor.java index 9df964c35..f3b6def65 100644 --- a/commonmark/src/main/java/org/commonmark/node/Visitor.java +++ b/commonmark/src/main/java/org/commonmark/node/Visitor.java @@ -7,8 +7,8 @@ */ public interface Visitor { - void visit(BlankLine blankLine); - + void visit(BlankLine blankLine); + void visit(BlockQuote blockQuote); void visit(BulletList bulletList); diff --git a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java index 618fd044b..76dc25355 100644 --- a/commonmark/src/main/java/org/commonmark/parser/SourceLines.java +++ b/commonmark/src/main/java/org/commonmark/parser/SourceLines.java @@ -66,8 +66,9 @@ public List getSourceSpans() { } /** - * Return original 
List<SourceLine&rt;
, but with first line removed. + * Return original 
List<SourceLine>
, but with first line removed. * If the list is already empty, return the list unchanged + * @return List of source lines, or a blank list */ public List removeFirstLine() { if(!isEmpty()) { diff --git a/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java b/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java index aa956a48a..7c8358f7a 100644 --- a/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/block/BlockParser.java @@ -13,18 +13,18 @@ public interface BlockParser { /** - * Return true if the block that is parsed is a container (contains other blocks), or false if it's a leaf. + * @return true if the block that is parsed is a container (contains other blocks), or false if it's a leaf. */ boolean isContainer(); /** - * Return true if the block can have lazy continuation lines. *

* Lazy continuation lines are lines that were rejected by this {@link #tryContinue(ParserState)} but didn't match * any other block parsers either. *

* If true is returned here, those lines will get added via {@link #addLine(SourceLine)}. For false, the block is * closed instead. + * @return true if the block can have lazy continuation lines. */ boolean canHaveLazyContinuationLines(); @@ -42,6 +42,7 @@ public interface BlockParser { * need to override this. * * @since 0.16.0 + * @param sourceSpan Source span to be added into the current block */ void addSourceSpan(SourceSpan sourceSpan); diff --git a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java index c1f3663e1..00b51890e 100644 --- a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java +++ b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterProcessor.java @@ -23,6 +23,7 @@ public interface DelimiterProcessor { /** * Minimum number of delimiter characters that are needed to activate this. Must be at least 1. + * @return the minimum number of delimiter characters needed to activate the processor. */ int getMinLength(); diff --git a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java index 578eac96b..dea13f4c9 100644 --- a/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java +++ b/commonmark/src/main/java/org/commonmark/parser/delimiter/DelimiterRun.java @@ -44,6 +44,8 @@ public interface DelimiterRun { *

* For example, for a delimiter run {@code ***}, calling this with 1 would return the last {@code *}. * Calling it with 2 would return the second last {@code *} and the last {@code *}. + * @param length Number of openers to capture + * @return Requested number of opening characters */ Iterable getOpeners(int length); @@ -53,6 +55,8 @@ public interface DelimiterRun { *

* For example, for a delimiter run {@code ***}, calling this with 1 would return the first {@code *}. * Calling it with 2 would return the first {@code *} and the second {@code *}. + * @param length Number of closers to capture + * @return Requested number of closing characters */ Iterable getClosers(int length); } diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java index 8fc5b683a..9d0fdbbd8 100644 --- a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CommonMarkWriter.java @@ -25,7 +25,7 @@ public void colon() { } public void line() { - append('\n'); + append('\n'); } public void writeStripped(String s) { diff --git a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java index 187a3c5ff..2f9707f05 100644 --- a/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java +++ b/commonmark/src/main/java/org/commonmark/renderer/roundtrip/CoreCommonMarkNodeRenderer.java @@ -87,21 +87,21 @@ public void render(Node node) { @Override public void visit(Document document) { // No rendering itself (aside from post-block whitespace, if present) - + visitChildren(document); - textContent.write(document.whitespacePostBlock()); + textContent.write(document.whitespaceEndOfDocument()); } public void visit(BlockQuote blockQuote) { - if(blockQuote.getFirstChild() instanceof Heading || - blockQuote.getFirstChild() instanceof IndentedCodeBlock || - blockQuote.getFirstChild() instanceof FencedCodeBlock) { - textContent.write(blockQuote.whitespacePreBlock()); - textContent.write(">"); - textContent.write(blockQuote.whitespacePreContent()); - } - + if(blockQuote.getFirstChild() instanceof Heading || + blockQuote.getFirstChild() instanceof IndentedCodeBlock || + blockQuote.getFirstChild() instanceof FencedCodeBlock) { + textContent.write(blockQuote.whitespacePreMarker()); + textContent.write(">"); + textContent.write(blockQuote.whitespacePostMarker()); + } + visitChildren(blockQuote); writeEndOfLineIfNeeded(blockQuote, null); @@ -134,7 +134,7 @@ public void visit(Code code) { @Override public void visit(Emphasis emphasis) { - textContent.write(emphasis.whitespacePreBlock()); + textContent.write(emphasis.whitespacePreBlock()); textContent.write(emphasis.getOpeningDelimiter()); visitChildren(emphasis); textContent.write(emphasis.getClosingDelimiter()); @@ -156,13 +156,13 @@ public void visit(FencedCodeBlock fencedCodeBlock) { // CommonMark test case #96 shows that content with no ending fence and // completely blank content should not trigger a new line if(fencedCodeBlock.getEndFenceLength() > 0 || !fencedCodeBlock.getRaw().isEmpty()) { - textContent.line(); + textContent.line(); } textContent.write(fencedCodeBlock.getRaw()); if(fencedCodeBlock.getEndFenceLength() > 0 && !fencedCodeBlock.getRaw().isEmpty()) { - textContent.line(); + textContent.line(); } for(int i = 0; i < fencedCodeBlock.getEndFenceIndent(); i++) { @@ -182,17 +182,17 @@ public void visit(FencedCodeBlock fencedCodeBlock) { @Override public void visit(HardLineBreak hardLineBreak) { - if(hardLineBreak.hasBackslash()) { - textContent.write("\\"); - } - + if(hardLineBreak.hasBackslash()) { + textContent.write("\\"); + } + writeEndOfLine(); } public void visit(Heading heading) { if(heading.getSymbolType() == '#') { - textContent.write(heading.whitespacePreBlock()); + textContent.write(heading.whitespacePreBlock()); for(int i = 0; i < heading.getLevel(); i++) { textContent.write(heading.getSymbolType()); @@ -215,33 +215,33 @@ public void visit(Heading heading) { writeEndOfLineIfNeeded(heading, null); }else { - textContent.write(heading.whitespacePreBlock()); - - visitChildren(heading); - - textContent.write(heading.whitespacePreContent()); - - writeEndOfLine(); - - textContent.write(heading.whitespacePostContent()); - + textContent.write(heading.whitespacePreBlock()); + + visitChildren(heading); + + textContent.write(heading.whitespacePreContent()); + + writeEndOfLine(); + + textContent.write(heading.whitespacePostContent()); + if(heading.getNumEndingSymbol() > 0) { - for(int i = 0; i < heading.getNumEndingSymbol(); i++) { - textContent.write(heading.getSymbolType()); - } + for(int i = 0; i < heading.getNumEndingSymbol(); i++) { + textContent.write(heading.getSymbolType()); + } } - - textContent.write(heading.whitespacePostBlock()); + + textContent.write(heading.whitespacePostBlock()); writeEndOfLineIfNeeded(heading, null); } } public void visit(ThematicBreak thematicBreak) { - if(thematicBreak.getContent() != null) { - textContent.write(thematicBreak.getContent().toString()); - } - + if(thematicBreak.getContent() != null) { + textContent.write(thematicBreak.getContent().toString()); + } + writeEndOfLineIfNeeded(thematicBreak, null); } @@ -252,8 +252,8 @@ public void visit(HtmlInline htmlInline) { @Override public void visit(HtmlBlock htmlBlock) { - textContent.write(htmlBlock.getRaw()); - writeEndOfLineIfNeeded(htmlBlock, null); + textContent.write(htmlBlock.getRaw()); + writeEndOfLineIfNeeded(htmlBlock, null); } @Override @@ -263,11 +263,11 @@ public void visit(Image image) { @Override public void visit(IndentedCodeBlock indentedCodeBlock) { - textContent.write(indentedCodeBlock.whitespacePreContent()); - - textContent.write(indentedCodeBlock.getRaw()); - - writeEndOfLineIfNeeded(indentedCodeBlock, null); + textContent.write(indentedCodeBlock.getIndentWhitespace()); + + textContent.write(indentedCodeBlock.getRaw()); + + writeEndOfLineIfNeeded(indentedCodeBlock, null); } public void visit(Link link) { @@ -285,38 +285,34 @@ public void visit(ListItem listItem) { if (listHolder != null && listHolder instanceof OrderedListHolder) { OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder; - textContent.write(listItem.whitespacePreBlock()); + textContent.write(listItem.whitespacePreMarker()); if(!(listItem.getFirstChild() instanceof ThematicBreak) && - !(listItem.getFirstChild() instanceof HtmlBlock) && - !(listItem.getParent().getParent() instanceof BlockQuote) && - !(listItem.getFirstChild() instanceof BlankLine)) { - textContent.write(listItem.getRawNumber() + ""); - textContent.write(orderedListHolder.getDelimiter()); - textContent.write(listItem.whitespacePreContent()); + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(listItem.getRawNumber() + ""); + textContent.write(orderedListHolder.getDelimiter()); + textContent.write(listItem.whitespacePostMarker()); } visitChildren(listItem); - textContent.write(listItem.whitespacePostBlock()); - writeEndOfLineIfNeeded(listItem, null); } else if (listHolder != null && listHolder instanceof BulletListHolder) { BulletListHolder bulletListHolder = (BulletListHolder) listHolder; - textContent.write(listItem.whitespacePreBlock()); + textContent.write(listItem.whitespacePreMarker()); if(!(listItem.getFirstChild() instanceof ThematicBreak) && - !(listItem.getFirstChild() instanceof HtmlBlock) && - !(listItem.getParent().getParent() instanceof BlockQuote) && - !(listItem.getFirstChild() instanceof BlankLine)) { - textContent.write(bulletListHolder.getMarker()); - textContent.write(listItem.whitespacePreContent()); + !(listItem.getFirstChild() instanceof HtmlBlock) && + !(listItem.getParent().getParent() instanceof BlockQuote) && + !(listItem.getFirstChild() instanceof BlankLine)) { + textContent.write(bulletListHolder.getMarker()); + textContent.write(listItem.whitespacePostMarker()); } visitChildren(listItem); - textContent.write(listItem.whitespacePostBlock()); - writeEndOfLineIfNeeded(listItem, null); } @@ -340,19 +336,17 @@ public void visit(Paragraph paragraph) { visitChildren(paragraph); writeEndOfLineIfNeeded(paragraph, null); - - textContent.write(paragraph.whitespacePostBlock()); } @Override public void visit(SoftLineBreak softLineBreak) { - // Within roundtrip rendering, a soft break is always a newline of some kind + // Within roundtrip rendering, a soft break is always a newline of some kind writeEndOfLine(); } @Override public void visit(StrongEmphasis strongEmphasis) { - textContent.write(strongEmphasis.whitespacePreBlock()); + textContent.write(strongEmphasis.whitespacePreBlock()); textContent.write(strongEmphasis.getOpeningDelimiter()); visitChildren(strongEmphasis); textContent.write(strongEmphasis.getClosingDelimiter()); @@ -360,21 +354,21 @@ public void visit(StrongEmphasis strongEmphasis) { @Override public void visit(Text text) { - if(!text.getRaw().isEmpty()) { - textContent.write(text.whitespacePreContent() + text.getRaw() + text.whitespacePostContent()); - }else { - textContent.write(text.whitespacePreContent() + text.getLiteral() + text.whitespacePostContent()); - } + if(!text.getRaw().isEmpty()) { + textContent.write(text.whitespacePreContent() + text.getRaw() + text.whitespacePostContent()); + }else { + textContent.write(text.whitespacePreContent() + text.getLiteral() + text.whitespacePostContent()); + } } @Override - public void visit(BlankLine blankLine) { - System.out.println(blankLine.getRaw()); - textContent.write(blankLine.getRaw()); - writeEndOfLineIfNeeded(blankLine, null); - } + public void visit(BlankLine blankLine) { + System.out.println(blankLine.getRaw()); + textContent.write(blankLine.getRaw()); + writeEndOfLineIfNeeded(blankLine, null); + } - @Override + @Override protected void visitChildren(Node parent) { Node node = parent.getFirstChild(); @@ -403,18 +397,18 @@ private void writeLink(Link node) { visitChildren(node); textContent.write(">"); }else if(node.getLinkType() == LinkType.REFERENCE) { - textContent.write("["); + textContent.write("["); visitChildren(node); textContent.write("]"); if(node.getLabel() != null) { - textContent.write("["); - if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { - textContent.write(node.getRawLabel()); - }else { - textContent.write(node.getLabel()); - } - textContent.write("]"); + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); } }else { textContent.write("["); @@ -424,8 +418,8 @@ private void writeLink(Link node) { textContent.write("("); if(!node.getRawDestination().isEmpty()) { - textContent.write(node.getWhitespacePreDestination()); - textContent.write(node.getRawDestination()); + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); }else if(node.getDestination() != null) { textContent.write(node.getWhitespacePreDestination()); textContent.write(node.getDestination()); @@ -434,8 +428,8 @@ private void writeLink(Link node) { } if(!node.getRawTitle().isEmpty()) { - textContent.write(node.getWhitespacePreTitle()); - textContent.write(node.getRawTitle()); + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); }else if(node.getTitle() != null) { textContent.write(node.getWhitespacePreTitle()); textContent.write(node.getTitleSymbol() + ""); @@ -443,9 +437,9 @@ private void writeLink(Link node) { textContent.write(node.getTitle()); if(node.getTitleSymbol() == '(') { - textContent.write(")"); + textContent.write(")"); }else { - textContent.write(node.getTitleSymbol()); + textContent.write(node.getTitleSymbol()); } } @@ -455,38 +449,38 @@ private void writeLink(Link node) { } private void writeLinkReference(LinkReferenceDefinition node) { - textContent.write(node.whitespacePreLabel()); - + textContent.write(node.whitespacePreLabel()); + textContent.write("["); textContent.write(node.getLabel()); textContent.write("]:"); if(node.getDestination() != null) { - textContent.write(node.whitespacePreDestination()); - if(!node.getRawDestination().isEmpty()) { - textContent.write(node.getRawDestination()); - }else { - textContent.write(node.getDestination()); - } + textContent.write(node.whitespacePreDestination()); + if(!node.getRawDestination().isEmpty()) { + textContent.write(node.getRawDestination()); + }else { + textContent.write(node.getDestination()); + } if(node.getTitle() != null) { - textContent.write(node.whitespacePreTitle()); - - char delimiter = node.getDelimiterChar(); - - if(delimiter == Character.MIN_VALUE) { - delimiter = '"'; - } - - textContent.write(delimiter); - - if(!node.getRawTitle().isEmpty()) { - textContent.write(node.getRawTitle()); - }else { - textContent.write(node.getTitle()); - } - - textContent.write(delimiter); + textContent.write(node.whitespacePreTitle()); + + char delimiter = node.getDelimiterChar(); + + if(delimiter == Character.MIN_VALUE) { + delimiter = '"'; + } + + textContent.write(delimiter); + + if(!node.getRawTitle().isEmpty()) { + textContent.write(node.getRawTitle()); + }else { + textContent.write(node.getTitle()); + } + + textContent.write(delimiter); } } @@ -500,18 +494,18 @@ private void writeImage(Image node) { visitChildren(node); textContent.write(">"); }else if(node.getLinkType() == LinkType.REFERENCE) { - textContent.write("!["); + textContent.write("!["); visitChildren(node); textContent.write("]"); if(node.getLabel() != null) { - textContent.write("["); - if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { - textContent.write(node.getRawLabel()); - }else { - textContent.write(node.getLabel()); - } - textContent.write("]"); + textContent.write("["); + if(node.getRawLabel() != null && !node.getRawLabel().isEmpty()) { + textContent.write(node.getRawLabel()); + }else { + textContent.write(node.getLabel()); + } + textContent.write("]"); } }else { textContent.write("!["); @@ -521,8 +515,8 @@ private void writeImage(Image node) { textContent.write("("); if(!node.getRawDestination().isEmpty()) { - textContent.write(node.getWhitespacePreDestination()); - textContent.write(node.getRawDestination()); + textContent.write(node.getWhitespacePreDestination()); + textContent.write(node.getRawDestination()); }else if(node.getDestination() != null) { textContent.write(node.getWhitespacePreDestination()); textContent.write(node.getDestination()); @@ -531,8 +525,8 @@ private void writeImage(Image node) { } if(!node.getRawTitle().isEmpty()) { - textContent.write(node.getWhitespacePreTitle()); - textContent.write(node.getRawTitle()); + textContent.write(node.getWhitespacePreTitle()); + textContent.write(node.getRawTitle()); }else if(node.getTitle() != null) { textContent.write(node.getWhitespacePreTitle()); textContent.write(node.getTitleSymbol() + ""); @@ -540,9 +534,9 @@ private void writeImage(Image node) { textContent.write(node.getTitle()); if(node.getTitleSymbol() == '(') { - textContent.write(")"); + textContent.write(")"); }else { - textContent.write(node.getTitleSymbol()); + textContent.write(node.getTitleSymbol()); } } @@ -553,19 +547,11 @@ private void writeImage(Image node) { private void writeEndOfLineIfNeeded(Node node, Character c) { if (node.getNext() != null) { - if(node instanceof ListItem) { - // If a list item has post-block whitespace, allow it to handle whitespace - if(((ListItem)node).whitespacePostBlock().isEmpty()) { - textContent.line(); - } - }else if(node instanceof Paragraph) { - // If a paragraph has post-block whitespace, allow it to handle whitespace - if(((Paragraph)node).whitespacePostBlock().isEmpty()) { - textContent.line(); - } - }else { - textContent.line(); - } + if(node instanceof ListItem || node instanceof Paragraph) { + textContent.line(); + }else { + textContent.line(); + } } } diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index 0dd759746..e486750aa 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -155,31 +155,6 @@ private String firstText(Node n) { } private static class DashBlock extends CustomBlock { - - @Override - public String whitespacePreBlock() { - return ""; - } - - @Override - public String whitespacePreContent() { - return ""; - } - - @Override - public String whitespacePostContent() { - return ""; - } - - @Override - public String whitespacePostBlock() { - return ""; - } - - @Override - public void setWhitespace(String... newWhitespace) { - // No whitespace needed for testing - } } private static class DashBlockParser extends AbstractBlockParser { From 6e6a81eb397a83b884a747800cacc223f54a49f4 Mon Sep 17 00:00:00 2001 From: Seii Date: Fri, 1 Oct 2021 11:41:32 -0600 Subject: [PATCH 5/5] Fix errors in Tables and Image Attributes extensions --- .../commonmark/ext/gfm/tables/internal/TableBlockParser.java | 2 +- .../internal/ImageAttributesDelimiterProcessor.java | 5 +++++ .../commonmark/internal/LinkReferenceDefinitionParser.java | 5 ----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java index a8eedb7a6..b978ef95b 100644 --- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java +++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java @@ -45,7 +45,7 @@ public BlockContinue tryContinue(ParserState state) { @Override public void addLine(SourceLine line) { - rowLines.add(line); + rowLines.add(line.getLiteralLine()); } @Override diff --git a/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java b/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java index a584948e3..9f3b6e9bb 100644 --- a/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java +++ b/commonmark-ext-image-attributes/src/main/java/org/commonmark/ext/image/attributes/internal/ImageAttributesDelimiterProcessor.java @@ -85,4 +85,9 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) { return 1; } + + @Override + public int process(DelimiterRun openingRun, DelimiterRun closingRun, String prefix) { + return process(openingRun, closingRun); + } } diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java index 866c6f4e7..8a8a23811 100644 --- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java @@ -51,11 +51,6 @@ public void parse(SourceLine line) { } Scanner scanner = Scanner.of(SourceLines.of(line)); - if(line.getSourceSpan() != null) { - // Ensure that LinkReferenceDefinition objects can have populated source spans - // Source spans track literal lines, not raw lines - addSourceSpan(line.getLiteralLine().getSourceSpan()); - } while (scanner.hasNext()) { boolean success;