diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java index 2952a8785..77bd61b3a 100644 --- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java +++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java @@ -1,20 +1,11 @@ package org.commonmark.ext.gfm.tables.internal; -import org.commonmark.ext.gfm.tables.TableBlock; -import org.commonmark.ext.gfm.tables.TableBody; -import org.commonmark.ext.gfm.tables.TableCell; -import org.commonmark.ext.gfm.tables.TableHead; -import org.commonmark.ext.gfm.tables.TableRow; +import org.commonmark.ext.gfm.tables.*; import org.commonmark.node.Block; import org.commonmark.node.Node; import org.commonmark.node.SourceSpan; import org.commonmark.parser.InlineParser; -import org.commonmark.parser.block.AbstractBlockParser; -import org.commonmark.parser.block.AbstractBlockParserFactory; -import org.commonmark.parser.block.BlockContinue; -import org.commonmark.parser.block.BlockStart; -import org.commonmark.parser.block.MatchedBlockParser; -import org.commonmark.parser.block.ParserState; +import org.commonmark.parser.block.*; import java.util.ArrayList; import java.util.Collections; @@ -116,7 +107,7 @@ private TableCell parseCell(CellSource cell, int column, InlineParser inlinePars tableCell.setSourceSpans(Collections.singletonList(cell.sourceSpan)); } - inlineParser.parse(cell.content, tableCell); + inlineParser.parse(Collections.singletonList(cell.content.trim()), tableCell); return tableCell; } @@ -246,11 +237,12 @@ public static class Factory extends AbstractBlockParserFactory { @Override public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { CharSequence line = state.getLine(); - CharSequence paragraph = matchedBlockParser.getParagraphContent(); - if (paragraph != null && paragraph.toString().contains("|") && !paragraph.toString().contains("\n")) { + List paragraphLines = matchedBlockParser.getParagraphLines(); + if (paragraphLines.size() == 1 && paragraphLines.get(0).toString().contains("|")) { CharSequence separatorLine = line.subSequence(state.getIndex(), line.length()); List columns = parseSeparator(separatorLine); if (columns != null && !columns.isEmpty()) { + CharSequence paragraph = paragraphLines.get(0); List headerCells = split(paragraph, null); if (columns.size() >= headerCells.size()) { return BlockStart.of(new TableBlockParser(columns, paragraph)) diff --git a/commonmark/src/main/java/org/commonmark/internal/Bracket.java b/commonmark/src/main/java/org/commonmark/internal/Bracket.java index 70a8a6e25..f66a79279 100644 --- a/commonmark/src/main/java/org/commonmark/internal/Bracket.java +++ b/commonmark/src/main/java/org/commonmark/internal/Bracket.java @@ -1,5 +1,6 @@ package org.commonmark.internal; +import org.commonmark.internal.inline.Position; import org.commonmark.node.Text; /** @@ -8,7 +9,10 @@ public class Bracket { public final Text node; - public final int index; + /** + * The position of the content (after the opening bracket) + */ + public final Position contentPosition; public final boolean image; /** @@ -31,17 +35,17 @@ public class Bracket { */ public boolean bracketAfter = false; - static public Bracket link(Text node, int index, Bracket previous, Delimiter previousDelimiter) { - return new Bracket(node, index, previous, previousDelimiter, false); + static public Bracket link(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter) { + return new Bracket(node, contentPosition, previous, previousDelimiter, false); } - static public Bracket image(Text node, int index, Bracket previous, Delimiter previousDelimiter) { - return new Bracket(node, index, previous, previousDelimiter, true); + static public Bracket image(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter) { + return new Bracket(node, contentPosition, previous, previousDelimiter, true); } - private Bracket(Text node, int index, Bracket previous, Delimiter previousDelimiter, boolean image) { + private Bracket(Text node, Position contentPosition, Bracket previous, Delimiter previousDelimiter, boolean image) { this.node = node; - this.index = index; + this.contentPosition = contentPosition; this.image = image; this.previous = previous; this.previousDelimiter = previousDelimiter; diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java index 51791d90b..58b390b63 100644 --- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java @@ -425,6 +425,8 @@ private void addLine() { } sb.append(rest); content = sb.toString(); + } else if (index == 0) { + content = line; } else { content = line.subSequence(index, line.length()); } @@ -458,9 +460,8 @@ private BlockStartImpl findBlockStart(BlockParser blockParser) { } /** - * Finalize a block. Close it and do any necessary postprocessing, e.g. creating string_content from strings, - * setting the 'tight' or 'loose' status of a list, and parsing the beginnings of paragraphs for reference - * definitions. + * Finalize a block. Close it and do any necessary postprocessing, e.g. setting the content of blocks and + * collecting link reference definitions from paragraphs. */ private void finalize(BlockParser blockParser) { if (blockParser instanceof ParagraphParser) { @@ -567,17 +568,12 @@ public BlockParser getMatchedBlockParser() { } @Override - public CharSequence getParagraphContent() { + public List getParagraphLines() { if (matchedBlockParser instanceof ParagraphParser) { ParagraphParser paragraphParser = (ParagraphParser) matchedBlockParser; - CharSequence content = paragraphParser.getContentString(); - if (content.length() == 0) { - return null; - } - - return content; + return Collections.unmodifiableList(paragraphParser.getParagraphLines()); } - return null; + return Collections.emptyList(); } } diff --git a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java index 2b72ba236..88bf002f0 100644 --- a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java @@ -6,12 +6,15 @@ import org.commonmark.parser.InlineParser; import org.commonmark.parser.block.*; +import java.util.Collections; +import java.util.List; + public class HeadingParser extends AbstractBlockParser { private final Heading block = new Heading(); - private final String content; + private final List content; - public HeadingParser(int level, String content) { + public HeadingParser(int level, List content) { block.setLevel(level); this.content = content; } @@ -49,10 +52,9 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar int setextHeadingLevel = getSetextHeadingLevel(line, nextNonSpace); if (setextHeadingLevel > 0) { - CharSequence paragraph = matchedBlockParser.getParagraphContent(); - if (paragraph != null) { - String content = paragraph.toString(); - return BlockStart.of(new HeadingParser(setextHeadingLevel, content)) + List paragraph = matchedBlockParser.getParagraphLines(); + if (!paragraph.isEmpty()) { + return BlockStart.of(new HeadingParser(setextHeadingLevel, paragraph)) .atIndex(line.length()) .replaceActiveBlockParser(); } @@ -73,25 +75,29 @@ private static HeadingParser getAtxHeading(CharSequence line, int index) { return null; } - int start = index + level; - if (start >= line.length()) { + int afterMarker = index + level; + if (afterMarker >= line.length()) { // End of line after markers is an empty heading - return new HeadingParser(level, ""); + return new HeadingParser(level, Collections.emptyList()); } - char next = line.charAt(start); + char next = line.charAt(afterMarker); if (!(next == ' ' || next == '\t')) { return null; } - int beforeSpace = Parsing.skipSpaceTabBackwards(line, line.length() - 1, start); - int beforeHash = Parsing.skipBackwards('#', line, beforeSpace, start); - int beforeTrailer = Parsing.skipSpaceTabBackwards(line, beforeHash, start); - if (beforeTrailer != beforeHash) { - return new HeadingParser(level, line.subSequence(start, beforeTrailer + 1).toString()); - } else { - return new HeadingParser(level, line.subSequence(start, beforeSpace + 1).toString()); + int start = Parsing.skipSpaceTab(line, afterMarker, line.length()); + + int beforeSpace = Parsing.skipSpaceTabBackwards(line, line.length() - 1, afterMarker); + int beforeHash = Parsing.skipBackwards('#', line, beforeSpace, afterMarker); + int beforeTrailer = Parsing.skipSpaceTabBackwards(line, beforeHash, afterMarker); + // Trailing `#` need to be separated with at least one space/tab, otherwise they are part of the content. + int end = (beforeTrailer < beforeHash) ? beforeTrailer + 1 : beforeSpace + 1; + if (start >= end) { + // Empty, e.g. `### ###` + return new HeadingParser(level, Collections.emptyList()); } + return new HeadingParser(level, Collections.singletonList(line.subSequence(start, end))); } // spec: A setext heading underline is a sequence of = characters or a sequence of - characters, with no more than diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java index dfb50149a..dedf68599 100644 --- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java +++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java @@ -1,9 +1,8 @@ package org.commonmark.internal; -import org.commonmark.internal.inline.AsteriskDelimiterProcessor; -import org.commonmark.internal.inline.UnderscoreDelimiterProcessor; +import org.commonmark.internal.inline.Scanner; +import org.commonmark.internal.inline.*; import org.commonmark.internal.util.Escaping; -import org.commonmark.internal.util.Html5Entities; import org.commonmark.internal.util.LinkScanner; import org.commonmark.internal.util.Parsing; import org.commonmark.node.*; @@ -12,53 +11,24 @@ import org.commonmark.parser.delimiter.DelimiterProcessor; import java.util.*; -import java.util.regex.Matcher; import java.util.regex.Pattern; -public class InlineParserImpl implements InlineParser { - - private static final String HTMLCOMMENT = "|"; - private static final String PROCESSINGINSTRUCTION = "[<][?].*?[?][>]"; - private static final String DECLARATION = "]*>"; - private static final String CDATA = ""; - private static final String HTMLTAG = "(?:" + Parsing.OPENTAG + "|" + Parsing.CLOSETAG + "|" + HTMLCOMMENT - + "|" + PROCESSINGINSTRUCTION + "|" + DECLARATION + "|" + CDATA + ")"; +public class InlineParserImpl implements InlineParser, InlineParserState { private static final String ASCII_PUNCTUATION = "!\"#\\$%&'\\(\\)\\*\\+,\\-\\./:;<=>\\?@\\[\\\\\\]\\^_`\\{\\|\\}~"; private static final Pattern PUNCTUATION = Pattern .compile("^[" + ASCII_PUNCTUATION + "\\p{Pc}\\p{Pd}\\p{Pe}\\p{Pf}\\p{Pi}\\p{Po}\\p{Ps}]"); - private static final Pattern HTML_TAG = Pattern.compile('^' + HTMLTAG, Pattern.CASE_INSENSITIVE); - - private static final Pattern ESCAPABLE = Pattern.compile('^' + Escaping.ESCAPABLE); - - private static final Pattern ENTITY_HERE = Pattern.compile('^' + Escaping.ENTITY, Pattern.CASE_INSENSITIVE); - - private static final Pattern TICKS = Pattern.compile("`+"); - - private static final Pattern TICKS_HERE = Pattern.compile("^`+"); - - private static final Pattern EMAIL_AUTOLINK = Pattern - .compile("^<([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>"); - - private static final Pattern AUTOLINK = Pattern - .compile("^<[a-zA-Z][a-zA-Z0-9.+-]{1,31}:[^<>\u0000-\u0020]*>"); - - private static final Pattern SPNL = Pattern.compile("^ *(?:\n *)?"); - private static final Pattern UNICODE_WHITESPACE_CHAR = Pattern.compile("^[\\p{Zs}\t\r\n\f]"); - private static final Pattern WHITESPACE = Pattern.compile("\\s+"); - - private static final Pattern FINAL_SPACE = Pattern.compile(" *$"); - private final BitSet specialCharacters; private final BitSet delimiterCharacters; private final Map delimiterProcessors; private final InlineParserContext context; + private final Map> inlineParsers; - private String input; - private int index; + private Scanner scanner; + private int trailingSpaces; /** * Top delimiter (emphasis, strong emphasis or custom emphasis). (Brackets are on a separate stack, different @@ -73,10 +43,16 @@ public class InlineParserImpl implements InlineParser { public InlineParserImpl(InlineParserContext inlineParserContext) { this.delimiterProcessors = calculateDelimiterProcessors(inlineParserContext.getCustomDelimiterProcessors()); - this.delimiterCharacters = calculateDelimiterCharacters(this.delimiterProcessors.keySet()); - this.specialCharacters = calculateSpecialCharacters(delimiterCharacters); this.context = inlineParserContext; + this.inlineParsers = new HashMap<>(); + this.inlineParsers.put('\\', Collections.singletonList(new BackslashInlineParser())); + this.inlineParsers.put('`', Collections.singletonList(new BackticksInlineParser())); + this.inlineParsers.put('&', Collections.singletonList(new EntityInlineParser())); + this.inlineParsers.put('<', Arrays.asList(new AutolinkInlineParser(), new HtmlInlineParser())); + + this.delimiterCharacters = calculateDelimiterCharacters(this.delimiterProcessors.keySet()); + this.specialCharacters = calculateSpecialCharacters(delimiterCharacters, inlineParsers.keySet()); } public static BitSet calculateDelimiterCharacters(Set characters) { @@ -87,17 +63,16 @@ public static BitSet calculateDelimiterCharacters(Set characters) { return bitSet; } - public static BitSet calculateSpecialCharacters(BitSet delimiterCharacters) { + public static BitSet calculateSpecialCharacters(BitSet delimiterCharacters, Set characters) { BitSet bitSet = new BitSet(); bitSet.or(delimiterCharacters); - bitSet.set('\n'); - bitSet.set('`'); + for (Character c : characters) { + bitSet.set(c); + } bitSet.set('['); bitSet.set(']'); - bitSet.set('\\'); bitSet.set('!'); - bitSet.set('<'); - bitSet.set('&'); + bitSet.set('\n'); return bitSet; } @@ -108,6 +83,11 @@ public static Map calculateDelimiterProcessors(Li return map; } + @Override + public Scanner scanner() { + return scanner; + } + private static void addDelimiterProcessors(Iterable delimiterProcessors, Map map) { for (DelimiterProcessor delimiterProcessor : delimiterProcessors) { char opening = delimiterProcessor.getOpeningCharacter(); @@ -142,16 +122,14 @@ private static void addDelimiterProcessorForChar(char delimiterChar, DelimiterPr } /** - * Parse content in block into inline children, using reference map to resolve references. + * Parse content in block into inline children, appending them to the block node. */ @Override - public void parse(String content, Node block) { - reset(content.trim()); + public void parse(List lines, Node block) { + reset(lines); - Node previous = null; while (true) { - Node node = parseInline(previous); - previous = node; + Node node = parseInline(); if (node != null) { block.appendChild(node); } else { @@ -163,200 +141,66 @@ public void parse(String content, Node block) { mergeChildTextNodes(block); } - void reset(String content) { - this.input = content; - this.index = 0; + void reset(List lines) { + this.scanner = Scanner.of(lines); + this.trailingSpaces = 0; this.lastDelimiter = null; this.lastBracket = null; } - - private Text text(String text, int beginIndex, int endIndex) { - return new Text(text.substring(beginIndex, endIndex)); - } - private Text text(String text) { return new Text(text); } /** - * Parse the next inline element in subject, advancing input index. + * Parse the next inline element in subject, advancing our position. * On success, return the new inline node. * On failure, return null. */ - private Node parseInline(Node previous) { - char c = peek(); - if (c == '\0') { + private Node parseInline() { + char c = scanner.peek(); + if (c == Scanner.END) { return null; } - Node node; - switch (c) { - case '\n': - node = parseNewline(previous); - break; - case '\\': - node = parseBackslash(); - break; - case '`': - node = parseBackticks(); - break; - case '[': - node = parseOpenBracket(); - break; - case '!': - node = parseBang(); - break; - case ']': - node = parseCloseBracket(); - break; - case '<': - node = parseAutolink(); - if (node == null) { - node = parseHtmlInline(); - } - break; - case '&': - node = parseEntity(); - break; - default: - boolean isDelimiter = delimiterCharacters.get(c); - if (isDelimiter) { - DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c); - node = parseDelimiters(delimiterProcessor, c); + Position position = scanner.position(); + List inlineParsers = this.inlineParsers.get(c); + if (inlineParsers != null) { + for (InlineContentParser inlineParser : inlineParsers) { + ParsedInline parsedInline = inlineParser.tryParse(this); + if (parsedInline instanceof ParsedInlineImpl) { + ParsedInlineImpl parsedInlineImpl = (ParsedInlineImpl) parsedInline; + scanner.setPosition(parsedInlineImpl.getPosition()); + return parsedInlineImpl.getNode(); } else { - node = parseString(); + // Reset position + scanner.setPosition(position); } - break; - } - if (node != null) { - return node; - } else { - index++; - // When we get here, it's only for a single special character that turned out to not have a special meaning. - // So we shouldn't have a single surrogate here, hence it should be ok to turn it into a String. - String literal = String.valueOf(c); - return text(literal); - } - } - - /** - * If RE matches at current index in the input, advance index and return the match; otherwise return null. - */ - private String match(Pattern re) { - if (index >= input.length()) { - return null; - } - try { - Matcher matcher = re.matcher(input); - matcher.region(index, input.length()); - boolean m = matcher.find(); - if (m) { - index = matcher.end(); - return matcher.group(); - } else { - return null; } - } catch (StackOverflowError e) { - return null; } - } - /** - * Returns the char at the current input index, or {@code '\0'} in case there are no more characters. - */ - private char peek() { - if (index < input.length()) { - return input.charAt(index); - } else { - return '\0'; + switch (c) { + case '[': + return parseOpenBracket(); + case '!': + return parseBang(); + case ']': + return parseCloseBracket(); + case '\n': + return parseLineBreak(); } - } - /** - * Parse zero or more space characters, including at most one newline. - */ - private void spnl() { - match(SPNL); - } - - /** - * Parse a newline. If it was preceded by two spaces, return a hard line break; otherwise a soft line break. - */ - private Node parseNewline(Node previous) { - index++; // assume we're at a \n - - // Check previous text for trailing spaces. - // The "endsWith" is an optimization to avoid an RE match in the common case. - if (previous instanceof Text && ((Text) previous).getLiteral().endsWith(" ")) { - Text text = (Text) previous; - String literal = text.getLiteral(); - Matcher matcher = FINAL_SPACE.matcher(literal); - int spaces = matcher.find() ? matcher.end() - matcher.start() : 0; - if (spaces > 0) { - text.setLiteral(literal.substring(0, literal.length() - spaces)); + boolean isDelimiter = delimiterCharacters.get(c); + if (isDelimiter) { + DelimiterProcessor delimiterProcessor = delimiterProcessors.get(c); + Node delimiterNode = parseDelimiters(delimiterProcessor, c); + if (delimiterNode != null) { + return delimiterNode; } - if (spaces >= 2) { - return new HardLineBreak(); - } else { - return new SoftLineBreak(); - } - } else { - return new SoftLineBreak(); } - } - /** - * Parse a backslash-escaped special character, adding either the escaped character, a hard line break - * (if the backslash is followed by a newline), or a literal backslash to the block's children. - */ - private Node parseBackslash() { - index++; - Node node; - if (peek() == '\n') { - node = new HardLineBreak(); - index++; - } else if (index < input.length() && ESCAPABLE.matcher(input.substring(index, index + 1)).matches()) { - node = text(input, index, index + 1); - index++; - } else { - node = text("\\"); - } - return node; - } - - /** - * Attempt to parse backticks, returning either a backtick code span or a literal sequence of backticks. - */ - private Node parseBackticks() { - String ticks = match(TICKS_HERE); - if (ticks == null) { - return null; - } - int afterOpenTicks = index; - String matched; - while ((matched = match(TICKS)) != null) { - if (matched.equals(ticks)) { - Code node = new Code(); - String content = input.substring(afterOpenTicks, index - ticks.length()); - content = content.replace('\n', ' '); - - // spec: If the resulting string both begins and ends with a space character, but does not consist - // entirely of space characters, a single space character is removed from the front and back. - if (content.length() >= 3 && - content.charAt(0) == ' ' && - content.charAt(content.length() - 1) == ' ' && - Parsing.hasNonSpace(content)) { - content = content.substring(1, content.length() - 1); - } - - node.setLiteral(content); - return node; - } - } - // If we got here, we didn't match a closing backtick sequence. - index = afterOpenTicks; - return text(ticks); + // If we get here, even for a special/delimiter character, we will just treat it as text. + return parseText(); } /** @@ -367,16 +211,13 @@ private Node parseDelimiters(DelimiterProcessor delimiterProcessor, char delimit if (res == null) { return null; } - int length = res.count; - int startIndex = index; - index += length; - Text node = text(input, startIndex, index); + Text node = res.text; // Add entry to stack for this opener lastDelimiter = new Delimiter(node, delimiterChar, res.canOpen, res.canClose, lastDelimiter); - lastDelimiter.length = length; - lastDelimiter.originalLength = length; + lastDelimiter.length = res.count; + lastDelimiter.originalLength = res.count; if (lastDelimiter.previous != null) { lastDelimiter.previous.next = lastDelimiter; } @@ -388,13 +229,13 @@ private Node parseDelimiters(DelimiterProcessor delimiterProcessor, char delimit * Add open bracket to delimiter stack and add a text node to block's children. */ private Node parseOpenBracket() { - int startIndex = index; - index++; + scanner.next(); + Position start = scanner.position(); Text node = text("["); // Add entry to stack for this opener - addBracket(Bracket.link(node, startIndex, lastBracket, lastDelimiter)); + addBracket(Bracket.link(node, start, lastBracket, lastDelimiter)); return node; } @@ -404,16 +245,12 @@ private Node parseOpenBracket() { * Otherwise just add a text node. */ private Node parseBang() { - int startIndex = index; - index++; - if (peek() == '[') { - index++; - + scanner.next(); + if (scanner.next('[')) { Text node = text("!["); // Add entry to stack for this opener - addBracket(Bracket.image(node, startIndex + 1, lastBracket, lastDelimiter)); - + addBracket(Bracket.image(node, scanner.position(), lastBracket, lastDelimiter)); return node; } else { return text("!"); @@ -425,8 +262,9 @@ private Node parseBang() { * plain [ character. If there is a matching delimiter, remove it from the delimiter stack. */ private Node parseCloseBracket() { - index++; - int startIndex = index; + Position beforeClose = scanner.position(); + scanner.next(); + Position afterClose = scanner.position(); // Get previous `[` or `![` Bracket opener = lastBracket; @@ -442,60 +280,59 @@ private Node parseCloseBracket() { } // Check to see if we have a link/image - String dest = null; String title = null; - boolean isLinkOrImage = false; // Maybe a inline link like `[foo](/uri "title")` - if (peek() == '(') { - index++; - spnl(); - if ((dest = parseLinkDestination()) != null) { - spnl(); + if (scanner.next('(')) { + scanner.whitespace(); + dest = parseLinkDestination(scanner); + if (dest == null) { + scanner.setPosition(afterClose); + } else { + int whitespace = scanner.whitespace(); // title needs a whitespace before - if (WHITESPACE.matcher(input.substring(index - 1, index)).matches()) { - title = parseLinkTitle(); - spnl(); + if (whitespace >= 1) { + title = parseLinkTitle(scanner); + scanner.whitespace(); } - if (peek() == ')') { - index++; - isLinkOrImage = true; - } else { - index = startIndex; + if (!scanner.next(')')) { + // Don't have a closing `)`, so it's not a destination and title -> reset. + // Note that something like `[foo](` could be valid, `(` will just be text. + scanner.setPosition(afterClose); + dest = null; + title = null; } } } - // Maybe a reference link like `[foo][bar]`, `[foo][]` or `[foo]` - if (!isLinkOrImage) { - + // Maybe a reference link like `[foo][bar]`, `[foo][]` or `[foo]`. + // Note that even `[foo](` could be a valid link if there's a reference, which is why this is not just an `else` + // here. + if (dest == null) { // See if there's a link label like `[bar]` or `[]` - int beforeLabel = index; - parseLinkLabel(); - int labelLength = index - beforeLabel; - String ref = null; - if (labelLength > 2) { - ref = input.substring(beforeLabel, beforeLabel + labelLength); - } else if (!opener.bracketAfter) { + String ref = parseLinkLabel(scanner); + if (ref == null) { + scanner.setPosition(afterClose); + } + if ((ref == null || ref.isEmpty()) && !opener.bracketAfter) { // If the second label is empty `[foo][]` or missing `[foo]`, then the first label is the reference. // But it can only be a reference when there's no (unescaped) bracket in it. // If there is, we don't even need to try to look up the reference. This is an optimization. - ref = input.substring(opener.index, startIndex); + ref = scanner.textBetween(opener.contentPosition, beforeClose).toString(); } if (ref != null) { - String label = Escaping.normalizeReference(ref); + String label = Escaping.normalizeLabelContent(ref); LinkReferenceDefinition definition = context.getLinkReferenceDefinition(label); if (definition != null) { dest = definition.getDestination(); title = definition.getTitle(); - isLinkOrImage = true; } } } - if (isLinkOrImage) { + if (dest != null) { // If we got here, open is a potential opener Node linkOrImage = opener.image ? new Image(dest, title) : new Link(dest, title); @@ -527,10 +364,11 @@ private Node parseCloseBracket() { return linkOrImage; - } else { // no link or image - index = startIndex; + } else { + // No link or image, parse just the bracket as text and continue removeLastBracket(); + scanner.setPosition(afterClose); return text("]"); } } @@ -549,124 +387,105 @@ private void removeLastBracket() { /** * Attempt to parse link destination, returning the string or null if no match. */ - private String parseLinkDestination() { - int afterDest = LinkScanner.scanLinkDestination(input, index); - if (afterDest == -1) { + private String parseLinkDestination(Scanner scanner) { + char delimiter = scanner.peek(); + Position start = scanner.position(); + if (!LinkScanner.scanLinkDestination(scanner)) { return null; } String dest; - if (peek() == '<') { + if (delimiter == '<') { // chop off surrounding <..>: - dest = input.substring(index + 1, afterDest - 1); + CharSequence rawDestination = scanner.textBetween(start, scanner.position()); + dest = rawDestination.subSequence(1, rawDestination.length() - 1).toString(); } else { - dest = input.substring(index, afterDest); + dest = scanner.textBetween(start, scanner.position()).toString(); } - index = afterDest; return Escaping.unescapeString(dest); } /** * Attempt to parse link title (sans quotes), returning the string or null if no match. */ - private String parseLinkTitle() { - int afterTitle = LinkScanner.scanLinkTitle(input, index); - if (afterTitle == -1) { + private String parseLinkTitle(Scanner scanner) { + Position start = scanner.position(); + if (!LinkScanner.scanLinkTitle(scanner)) { return null; } // chop off ', " or parens - String title = input.substring(index + 1, afterTitle - 1); - index = afterTitle; + CharSequence rawTitle = scanner.textBetween(start, scanner.position()); + String title = rawTitle.subSequence(1, rawTitle.length() - 1).toString(); return Escaping.unescapeString(title); } /** - * Attempt to parse a link label, returning number of characters parsed. + * Attempt to parse a link label, returning the label between the brackets or null. */ - int parseLinkLabel() { - if (index >= input.length() || input.charAt(index) != '[') { - return 0; + String parseLinkLabel(Scanner scanner) { + if (!scanner.next('[')) { + return null; } - int startContent = index + 1; - int endContent = LinkScanner.scanLinkLabelContent(input, startContent); - // spec: A link label can have at most 999 characters inside the square brackets. - int contentLength = endContent - startContent; - if (endContent == -1 || contentLength > 999) { - return 0; - } - if (endContent >= input.length() || input.charAt(endContent) != ']') { - return 0; + Position start = scanner.position(); + if (!LinkScanner.scanLinkLabelContent(scanner)) { + return null; } - index = endContent + 1; - return contentLength + 2; - } + Position end = scanner.position(); - /** - * Attempt to parse an autolink (URL or email in pointy brackets). - */ - private Node parseAutolink() { - String m; - if ((m = match(EMAIL_AUTOLINK)) != null) { - String dest = m.substring(1, m.length() - 1); - Link node = new Link("mailto:" + dest, null); - node.appendChild(new Text(dest)); - return node; - } else if ((m = match(AUTOLINK)) != null) { - String dest = m.substring(1, m.length() - 1); - Link node = new Link(dest, null); - node.appendChild(new Text(dest)); - return node; - } else { + if (!scanner.next(']')) { return null; } - } - /** - * Attempt to parse inline HTML. - */ - private Node parseHtmlInline() { - String m = match(HTML_TAG); - if (m != null) { - HtmlInline node = new HtmlInline(); - node.setLiteral(m); - return node; - } else { + String content = scanner.textBetween(start, end).toString(); + // spec: A link label can have at most 999 characters inside the square brackets. + if (content.length() > 999) { return null; } + + return content; } - /** - * Attempt to parse a HTML style entity. - */ - private Node parseEntity() { - String m; - if ((m = match(ENTITY_HERE)) != null) { - return text(Html5Entities.entityToString(m)); + private Node parseLineBreak() { + scanner.next(); + + if (trailingSpaces >= 2) { + return new HardLineBreak(); } else { - return null; + return new SoftLineBreak(); } } /** - * Parse a run of ordinary characters, or a single character with a special meaning in markdown, as a plain string. + * Parse the next character as plain text, and possibly more if the following characters are non-special. */ - private Node parseString() { - int begin = index; - int length = input.length(); - while (index != length) { - if (specialCharacters.get(input.charAt(index))) { + private Node parseText() { + Position start = scanner.position(); + scanner.next(); + while (scanner.hasNext()) { + if (specialCharacters.get(scanner.peek())) { break; } - index++; + scanner.next(); } - if (begin != index) { - return text(input, begin, index); - } else { - return null; + + String text = scanner.textBetween(start, scanner.position()).toString(); + + char c = scanner.peek(); + if (c == '\n') { + // We parsed until the end of the line. Trim any trailing spaces and remember them (for hard line breaks). + int end = Parsing.skipBackwards(' ', text, text.length() - 1, 0) + 1; + trailingSpaces = text.length() - end; + text = text.substring(0, end); + } else if (c == Scanner.END) { + // For the last line, both tabs and spaces are trimmed for some reason (checked with commonmark.js). + int end = Parsing.skipSpaceTabBackwards(text, text.length() - 1, 0) + 1; + text = text.substring(0, end); } + + return text(text); } /** @@ -676,25 +495,19 @@ private Node parseString() { * @return information about delimiter run, or {@code null} */ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char delimiterChar) { - int startIndex = index; + char charBefore = scanner.peekPrevious(); + Position start = scanner.position(); - int delimiterCount = 0; - while (peek() == delimiterChar) { - delimiterCount++; - index++; - } + int delimiterCount = scanner.matchMultiple(delimiterChar); if (delimiterCount < delimiterProcessor.getMinLength()) { - index = startIndex; + scanner.setPosition(start); return null; } - String before = startIndex == 0 ? "\n" : - input.substring(startIndex - 1, startIndex); - - char charAfter = peek(); - String after = charAfter == '\0' ? "\n" : - String.valueOf(charAfter); + char charAfter = scanner.peek(); + String before = charBefore == Scanner.END ? "\n" : String.valueOf(charBefore); + String after = charAfter == Scanner.END ? "\n" : String.valueOf(charAfter); // We could be more lazy here, in most cases we don't need to do every match case. boolean beforeIsPunctuation = PUNCTUATION.matcher(before).matches(); @@ -716,8 +529,8 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char canClose = rightFlanking && delimiterChar == delimiterProcessor.getClosingCharacter(); } - index = startIndex; - return new DelimiterData(delimiterCount, canOpen, canClose); + String text = scanner.textBetween(start, scanner.position()).toString(); + return new DelimiterData(delimiterCount, canOpen, canClose, new Text(text)); } private void processDelimiters(Delimiter stackBottom) { @@ -921,11 +734,13 @@ private static class DelimiterData { final int count; final boolean canClose; final boolean canOpen; + final Text text; - DelimiterData(int count, boolean canOpen, boolean canClose) { + DelimiterData(int count, boolean canOpen, boolean canClose, Text text) { this.count = count; this.canOpen = canOpen; this.canClose = canClose; + this.text = text; } } } diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java index e0c23160e..950fc31c8 100644 --- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java @@ -1,8 +1,9 @@ package org.commonmark.internal; +import org.commonmark.internal.inline.Position; +import org.commonmark.internal.inline.Scanner; import org.commonmark.internal.util.Escaping; import org.commonmark.internal.util.LinkScanner; -import org.commonmark.internal.util.Parsing; import org.commonmark.node.LinkReferenceDefinition; import org.commonmark.node.SourceSpan; @@ -18,7 +19,7 @@ public class LinkReferenceDefinitionParser { private State state = State.START_DEFINITION; - private final StringBuilder paragraph = new StringBuilder(); + private final List paragraphLines = new ArrayList<>(); private final List definitions = new ArrayList<>(); private final List sourceSpans = new ArrayList<>(); @@ -30,42 +31,43 @@ public class LinkReferenceDefinitionParser { private boolean referenceValid = false; public void parse(CharSequence line) { - if (paragraph.length() != 0) { - paragraph.append('\n'); + paragraphLines.add(line); + if (state == State.PARAGRAPH) { + // We're in a paragraph now. Link reference definitions can only appear at the beginning, so once + // we're in a paragraph, there's no going back. + return; } - paragraph.append(line); - int i = 0; - while (i < line.length()) { + Scanner scanner = Scanner.of(line); + while (scanner.hasNext()) { + boolean success; switch (state) { - case PARAGRAPH: { - // We're in a paragraph now. Link reference definitions can only appear at the beginning, so once - // we're in a paragraph, there's no going back. - return; - } case START_DEFINITION: { - i = startDefinition(line, i); + success = startDefinition(scanner); break; } case LABEL: { - i = label(line, i); + success = label(scanner); break; } case DESTINATION: { - i = destination(line, i); + success = destination(scanner); break; } case START_TITLE: { - i = startTitle(line, i); + success = startTitle(scanner); break; } case TITLE: { - i = title(line, i); + success = title(scanner); break; } + default: { + throw new IllegalStateException("Unknown parsing state: " + state); + } } - // -1 is returned if parsing failed, which means we fall back to treating text as a paragraph. - if (i == -1) { + // Parsing failed, which means we fall back to treating text as a paragraph. + if (!success) { state = State.PARAGRAPH; return; } @@ -76,8 +78,11 @@ public void addSourceSpan(SourceSpan sourceSpan) { sourceSpans.add(sourceSpan); } - CharSequence getParagraphContent() { - return paragraph; + /** + * @return the lines that are normal paragraph content, without newlines + */ + List getParagraphLines() { + return paragraphLines; } List getParagraphSourceSpans() { @@ -93,96 +98,95 @@ State getState() { return state; } - private int startDefinition(CharSequence line, int i) { - i = Parsing.skipSpaceTab(line, i, line.length()); - if (i >= line.length() || line.charAt(i) != '[') { - return -1; + private boolean startDefinition(Scanner scanner) { + scanner.whitespace(); + if (!scanner.next('[')) { + return false; } state = State.LABEL; label = new StringBuilder(); - int labelStart = i + 1; - if (labelStart >= line.length()) { + if (!scanner.hasNext()) { label.append('\n'); } - - return labelStart; + return true; } - private int label(CharSequence line, int i) { - int afterLabel = LinkScanner.scanLinkLabelContent(line, i); - if (afterLabel == -1) { - return -1; + private boolean label(Scanner scanner) { + Position start = scanner.position(); + if (!LinkScanner.scanLinkLabelContent(scanner)) { + return false; } - label.append(line, i, afterLabel); + label.append(scanner.textBetween(start, scanner.position())); - if (afterLabel >= line.length()) { + if (!scanner.hasNext()) { // label might continue on next line label.append('\n'); - return afterLabel; - } else if (line.charAt(afterLabel) == ']') { - int colon = afterLabel + 1; + return true; + } else if (scanner.next(']')) { // end of label - if (colon >= line.length() || line.charAt(colon) != ':') { - return -1; + if (!scanner.next(':')) { + return false; } // spec: A link label can have at most 999 characters inside the square brackets. if (label.length() > 999) { - return -1; + return false; } String normalizedLabel = Escaping.normalizeLabelContent(label.toString()); if (normalizedLabel.isEmpty()) { - return -1; + return false; } this.normalizedLabel = normalizedLabel; state = State.DESTINATION; - return Parsing.skipSpaceTab(line, colon + 1, line.length()); + scanner.whitespace(); + return true; } else { - return -1; + return false; } } - private int destination(CharSequence line, int i) { - i = Parsing.skipSpaceTab(line, i, line.length()); - int afterDestination = LinkScanner.scanLinkDestination(line, i); - if (afterDestination == -1) { - return -1; + private boolean destination(Scanner scanner) { + scanner.whitespace(); + Position start = scanner.position(); + if (!LinkScanner.scanLinkDestination(scanner)) { + return false; } - destination = (line.charAt(i) == '<') - ? line.subSequence(i + 1, afterDestination - 1).toString() - : line.subSequence(i, afterDestination).toString(); + String rawDestination = scanner.textBetween(start, scanner.position()).toString(); + destination = rawDestination.startsWith("<") ? + rawDestination.substring(1, rawDestination.length() - 1) : + rawDestination; - int afterSpace = Parsing.skipSpaceTab(line, afterDestination, line.length()); - if (afterSpace >= line.length()) { + int whitespace = scanner.whitespace(); + if (!scanner.hasNext()) { // Destination was at end of line, so this is a valid reference for sure (and maybe a title). // If not at end of line, wait for title to be valid first. referenceValid = true; - paragraph.setLength(0); - } else if (afterSpace == afterDestination) { + paragraphLines.clear(); + } else if (whitespace == 0) { // spec: The title must be separated from the link destination by whitespace - return -1; + return false; } state = State.START_TITLE; - return afterSpace; + return true; } - private int startTitle(CharSequence line, int i) { - i = Parsing.skipSpaceTab(line, i, line.length()); - if (i >= line.length()) { + private boolean startTitle(Scanner scanner) { + scanner.whitespace(); + if (!scanner.hasNext()) { state = State.START_DEFINITION; - return i; + return true; } titleDelimiter = '\0'; - char c = line.charAt(i); + char c = scanner.peek(); switch (c) { case '"': case '\'': @@ -196,8 +200,8 @@ private int startTitle(CharSequence line, int i) { if (titleDelimiter != '\0') { state = State.TITLE; title = new StringBuilder(); - i++; - if (i == line.length()) { + scanner.next(); + if (!scanner.hasNext()) { title.append('\n'); } } else { @@ -205,37 +209,38 @@ private int startTitle(CharSequence line, int i) { // There might be another reference instead, try that for the same character. state = State.START_DEFINITION; } - return i; + return true; } - private int title(CharSequence line, int i) { - int afterTitle = LinkScanner.scanLinkTitleContent(line, i, titleDelimiter); - if (afterTitle == -1) { + private boolean title(Scanner scanner) { + Position start = scanner.position(); + if (!LinkScanner.scanLinkTitleContent(scanner, titleDelimiter)) { // Invalid title, stop - return -1; + return false; } - title.append(line.subSequence(i, afterTitle)); + title.append(scanner.textBetween(start, scanner.position())); - if (afterTitle >= line.length()) { - // Title still going, continue on next line + if (!scanner.hasNext()) { + // Title ran until the end of line, so continue on next line (until we find the delimiter) title.append('\n'); - return afterTitle; + return true; } - int afterTitleDelimiter = afterTitle + 1; - int afterSpace = Parsing.skipSpaceTab(line, afterTitleDelimiter, line.length()); - if (afterSpace != line.length()) { + // Skip delimiter character + scanner.next(); + scanner.whitespace(); + if (scanner.hasNext()) { // spec: No further non-whitespace characters may occur on the line. - return -1; + return false; } referenceValid = true; finishReference(); - paragraph.setLength(0); + paragraphLines.clear(); // See if there's another definition. state = State.START_DEFINITION; - return afterSpace; + return true; } private void finishReference() { diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java index de1558f92..1538ca41b 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java @@ -210,7 +210,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar } int markerIndex = state.getNextNonSpaceIndex(); int markerColumn = state.getColumn() + state.getIndent(); - boolean inParagraph = matchedBlockParser.getParagraphContent() != null; + boolean inParagraph = !matchedBlockParser.getParagraphLines().isEmpty(); ListData listData = parseList(state.getLine(), markerIndex, markerColumn, inParagraph); if (listData == null) { return BlockStart.none(); diff --git a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java index 7e28cf0fb..8962ec7da 100644 --- a/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java +++ b/commonmark/src/main/java/org/commonmark/internal/ParagraphParser.java @@ -49,7 +49,7 @@ public void addSourceSpan(SourceSpan sourceSpan) { @Override public void closeBlock() { - if (linkReferenceDefinitionParser.getParagraphContent().length() == 0) { + if (linkReferenceDefinitionParser.getParagraphLines().isEmpty()) { block.unlink(); } else { block.setSourceSpans(linkReferenceDefinitionParser.getParagraphSourceSpans()); @@ -58,14 +58,14 @@ public void closeBlock() { @Override public void parseInlines(InlineParser inlineParser) { - CharSequence content = linkReferenceDefinitionParser.getParagraphContent(); - if (content.length() > 0) { - inlineParser.parse(content.toString(), block); + List lines = linkReferenceDefinitionParser.getParagraphLines(); + if (!lines.isEmpty()) { + inlineParser.parse(lines, block); } } - public CharSequence getContentString() { - return linkReferenceDefinitionParser.getParagraphContent(); + public List getParagraphLines() { + return linkReferenceDefinitionParser.getParagraphLines(); } public List getDefinitions() { diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java new file mode 100644 index 000000000..7b62d1257 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java @@ -0,0 +1,40 @@ +package org.commonmark.internal.inline; + +import org.commonmark.node.Link; +import org.commonmark.node.Node; +import org.commonmark.node.Text; + +import java.util.regex.Pattern; + +/** + * Attempt to parse an autolink (URL or email in pointy brackets). + */ +public class AutolinkInlineParser implements InlineContentParser { + + private static final Pattern URI = Pattern + .compile("^[a-zA-Z][a-zA-Z0-9.+-]{1,31}:[^<>\u0000-\u0020]*$"); + + private static final Pattern EMAIL = Pattern + .compile("^([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)$"); + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + Scanner scanner = inlineParserState.scanner(); + scanner.next(); + Position start = scanner.position(); + if (scanner.find('>') > 0) { + String text = scanner.textBetween(start, scanner.position()).toString(); + scanner.next(); + if (URI.matcher(text).matches()) { + Link node = new Link(text, null); + node.appendChild(new Text(text)); + return ParsedInline.of(node, scanner.position()); + } else if (EMAIL.matcher(text).matches()) { + Link node = new Link("mailto:" + text, null); + node.appendChild(new Text(text)); + return ParsedInline.of(node, scanner.position()); + } + } + return ParsedInline.none(); + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java new file mode 100644 index 000000000..f57a67a74 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java @@ -0,0 +1,35 @@ +package org.commonmark.internal.inline; + +import org.commonmark.internal.util.Escaping; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Node; +import org.commonmark.node.Text; + +import java.util.regex.Pattern; + +/** + * Parse a backslash-escaped special character, adding either the escaped character, a hard line break + * (if the backslash is followed by a newline), or a literal backslash to the block's children. + */ +public class BackslashInlineParser implements InlineContentParser { + + private static final Pattern ESCAPABLE = Pattern.compile('^' + Escaping.ESCAPABLE); + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + Scanner scanner = inlineParserState.scanner(); + // Backslash + scanner.next(); + + char next = scanner.peek(); + if (next == '\n') { + scanner.next(); + return ParsedInline.of(new HardLineBreak(), scanner.position()); + } else if (ESCAPABLE.matcher(String.valueOf(next)).matches()) { + scanner.next(); + return ParsedInline.of(new Text(String.valueOf(next)), scanner.position()); + } else { + return ParsedInline.of(new Text("\\"), scanner.position()); + } + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java new file mode 100644 index 000000000..f00079793 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java @@ -0,0 +1,47 @@ +package org.commonmark.internal.inline; + +import org.commonmark.internal.util.Parsing; +import org.commonmark.node.Code; +import org.commonmark.node.Node; +import org.commonmark.node.Text; + +/** + * Attempt to parse backticks, returning either a backtick code span or a literal sequence of backticks. + */ +public class BackticksInlineParser implements InlineContentParser { + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + Scanner scanner = inlineParserState.scanner(); + Position start = scanner.position(); + int openingTicks = scanner.matchMultiple('`'); + Position afterOpening = scanner.position(); + + while (scanner.find('`') > 0) { + Position beforeClosing = scanner.position(); + int count = scanner.matchMultiple('`'); + if (count == openingTicks) { + Code node = new Code(); + + String content = scanner.textBetween(afterOpening, beforeClosing).toString(); + content = content.replace('\n', ' '); + + // spec: If the resulting string both begins and ends with a space character, but does not consist + // entirely of space characters, a single space character is removed from the front and back. + if (content.length() >= 3 && + content.charAt(0) == ' ' && + content.charAt(content.length() - 1) == ' ' && + Parsing.hasNonSpace(content)) { + content = content.substring(1, content.length() - 1); + } + + node.setLiteral(content); + return ParsedInline.of(node, scanner.position()); + } + } + + // If we got here, we didn't find a matching closing backtick sequence. + String ticks = scanner.textBetween(start, afterOpening).toString(); + return ParsedInline.of(new Text(ticks), afterOpening); + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java new file mode 100644 index 000000000..f0c330002 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java @@ -0,0 +1,54 @@ +package org.commonmark.internal.inline; + +import org.commonmark.internal.util.AsciiMatcher; +import org.commonmark.internal.util.Html5Entities; +import org.commonmark.node.Node; +import org.commonmark.node.Text; + +/** + * Attempts to parse a HTML entity or numeric character reference. + */ +public class EntityInlineParser implements InlineContentParser { + + private static final AsciiMatcher hex = AsciiMatcher.builder().range('0', '9').range('A', 'F').range('a', 'f').build(); + private static final AsciiMatcher dec = AsciiMatcher.builder().range('0', '9').build(); + private static final AsciiMatcher entityStart = AsciiMatcher.builder().range('A', 'Z').range('a', 'z').build(); + private static final AsciiMatcher entityContinue = entityStart.newBuilder().range('0', '9').build(); + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + Scanner scanner = inlineParserState.scanner(); + Position start = scanner.position(); + // Skip `&` + scanner.next(); + + char c = scanner.peek(); + if (c == '#') { + // Numeric + scanner.next(); + if (scanner.next('x') || scanner.next('X')) { + int digits = scanner.match(hex); + if (1 <= digits && digits <= 6 && scanner.next(';')) { + return entity(scanner, start); + } + } else { + int digits = scanner.match(dec); + if (1 <= digits && digits <= 7 && scanner.next(';')) { + return entity(scanner, start); + } + } + } else if (entityStart.matches(c)) { + scanner.match(entityContinue); + if (scanner.next(';')) { + return entity(scanner, start); + } + } + + return ParsedInline.none(); + } + + private ParsedInline entity(Scanner scanner, Position start) { + String text = scanner.textBetween(start, scanner.position()).toString(); + return ParsedInline.of(new Text(Html5Entities.entityToString(text)), scanner.position()); + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java new file mode 100644 index 000000000..1fe20ecb8 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java @@ -0,0 +1,202 @@ +package org.commonmark.internal.inline; + +import org.commonmark.internal.util.AsciiMatcher; +import org.commonmark.node.HtmlInline; + +/** + * Attempt to parse inline HTML. + */ +public class HtmlInlineParser implements InlineContentParser { + + private static final AsciiMatcher asciiLetter = AsciiMatcher.builder().range('A', 'Z').range('a', 'z').build(); + + // spec: A tag name consists of an ASCII letter followed by zero or more ASCII letters, digits, or hyphens (-). + private static final AsciiMatcher tagNameStart = asciiLetter; + private static final AsciiMatcher tagNameContinue = tagNameStart.newBuilder().range('0', '9').c('-').build(); + + // spec: An attribute name consists of an ASCII letter, _, or :, followed by zero or more ASCII letters, digits, + // _, ., :, or -. (Note: This is the XML specification restricted to ASCII. HTML5 is laxer.) + private static final AsciiMatcher attributeStart = asciiLetter.newBuilder().c('_').c(':').build(); + private static final AsciiMatcher attributeContinue = attributeStart.newBuilder().range('0', '9').c('.').c('-').build(); + // spec: An unquoted attribute value is a nonempty string of characters not including whitespace, ", ', =, <, >, or `. + private static final AsciiMatcher attributeValueEnd = AsciiMatcher.builder() + .c(' ').c('\t').c('\n').c('\u000B').c('\f').c('\r') + .c('"').c('\'').c('=').c('<').c('>').c('`') + .build(); + + private static final AsciiMatcher declaration = AsciiMatcher.builder().range('A', 'Z').build(); + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + Scanner scanner = inlineParserState.scanner(); + Position start = scanner.position(); + // Skip over `<` + scanner.next(); + + char c = scanner.peek(); + if (tagNameStart.matches(c)) { + if (tryOpenTag(scanner)) { + return htmlInline(start, scanner); + } + } else if (c == '/') { + if (tryClosingTag(scanner)) { + return htmlInline(start, scanner); + } + } else if (c == '?') { + if (tryProcessingInstruction(scanner)) { + return htmlInline(start, scanner); + } + } else if (c == '!') { + // comment, declaration or CDATA + scanner.next(); + c = scanner.peek(); + if (c == '-') { + if (tryComment(scanner)) { + return htmlInline(start, scanner); + } + } else if (c == '[') { + if (tryCdata(scanner)) { + return htmlInline(start, scanner); + } + } else if (declaration.matches(c)) { + if (tryDeclaration(scanner)) { + return htmlInline(start, scanner); + } + } + } + + return ParsedInline.none(); + } + + private static ParsedInline htmlInline(Position start, Scanner scanner) { + String text = scanner.textBetween(start, scanner.position()).toString(); + HtmlInline node = new HtmlInline(); + node.setLiteral(text); + return ParsedInline.of(node, scanner.position()); + } + + private static boolean tryOpenTag(Scanner scanner) { + // spec: An open tag consists of a < character, a tag name, zero or more attributes, optional whitespace, + // an optional / character, and a > character. + scanner.next(); + scanner.match(tagNameContinue); + boolean whitespace = scanner.whitespace() >= 1; + // spec: An attribute consists of whitespace, an attribute name, and an optional attribute value specification. + while (whitespace && scanner.match(attributeStart) >= 1) { + scanner.match(attributeContinue); + // spec: An attribute value specification consists of optional whitespace, a = character, + // optional whitespace, and an attribute value. + whitespace = scanner.whitespace() >= 1; + if (scanner.next('=')) { + scanner.whitespace(); + char valueStart = scanner.peek(); + if (valueStart == '\'') { + scanner.next(); + if (scanner.find('\'') < 0) { + return false; + } + scanner.next(); + } else if (valueStart == '"') { + scanner.next(); + if (scanner.find('"') < 0) { + return false; + } + scanner.next(); + } else { + if (scanner.find(attributeValueEnd) <= 0) { + return false; + } + } + + // Whitespace is required between attributes + whitespace = scanner.whitespace() >= 1; + } + } + + scanner.next('/'); + return scanner.next('>'); + } + + private static boolean tryClosingTag(Scanner scanner) { + // spec: A closing tag consists of the string . + scanner.next(); + if (scanner.match(tagNameStart) >= 1) { + scanner.match(tagNameContinue); + scanner.whitespace(); + return scanner.next('>'); + } + return false; + } + + private static boolean tryProcessingInstruction(Scanner scanner) { + // spec: A processing instruction consists of the string , + // and the string ?>. + scanner.next(); + while (scanner.find('?') > 0) { + scanner.next(); + if (scanner.next('>')) { + return true; + } + } + return false; + } + + private static boolean tryComment(Scanner scanner) { + // spec: An HTML comment consists of , where text does not start with > or ->, does not end + // with -, and does not contain --. (See the HTML5 spec.) + + // Skip first `-` + scanner.next(); + if (!scanner.next('-')) { + return false; + } + + if (scanner.next('>') || scanner.next("->")) { + return false; + } + + while (scanner.find('-') >= 0) { + if (scanner.next("--")) { + return scanner.next('>'); + } else { + scanner.next(); + } + } + + return false; + } + + private static boolean tryCdata(Scanner scanner) { + // spec: A CDATA section consists of the string , + // and the string ]]>. + + // Skip `[` + scanner.next(); + + if (scanner.next("CDATA[")) { + while (scanner.find(']') >= 0) { + if (scanner.next("]]>")) { + return true; + } else { + scanner.next(); + } + } + } + + return false; + } + + private static boolean tryDeclaration(Scanner scanner) { + // spec: A declaration consists of the string , and the character >. + scanner.match(declaration); + if (scanner.whitespace() <= 0) { + return false; + } + if (scanner.find('>') >= 0) { + scanner.next(); + return true; + } + return false; + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java new file mode 100644 index 000000000..755ee3135 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/InlineContentParser.java @@ -0,0 +1,6 @@ +package org.commonmark.internal.inline; + +public interface InlineContentParser { + + ParsedInline tryParse(InlineParserState inlineParserState); +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java b/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java new file mode 100644 index 000000000..f6cb6bf49 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java @@ -0,0 +1,13 @@ +package org.commonmark.internal.inline; + +public interface InlineParserState { + + /** + * Return a scanner for the input for the current position (on the character that the inline parser registered + * interest for). + *

+ * Note that this always returns the same instance, if you want to backtrack you need to use + * {@link Scanner#position()} and {@link Scanner#setPosition(Position)}. + */ + Scanner scanner(); +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java new file mode 100644 index 000000000..7e6ece88e --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java @@ -0,0 +1,23 @@ +package org.commonmark.internal.inline; + +import org.commonmark.node.Node; + +public abstract class ParsedInline { + + protected ParsedInline() { + } + + public static ParsedInline none() { + return null; + } + + public static ParsedInline of(Node node, Position position) { + if (node == null) { + throw new NullPointerException("node must not be null"); + } + if (position == null) { + throw new NullPointerException("position must not be null"); + } + return new ParsedInlineImpl(node, position); + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java new file mode 100644 index 000000000..aea325f27 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java @@ -0,0 +1,21 @@ +package org.commonmark.internal.inline; + +import org.commonmark.node.Node; + +public class ParsedInlineImpl extends ParsedInline { + private final Node node; + private final Position position; + + ParsedInlineImpl(Node node, Position position) { + this.node = node; + this.position = position; + } + + public Node getNode() { + return node; + } + + public Position getPosition() { + return position; + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Position.java b/commonmark/src/main/java/org/commonmark/internal/inline/Position.java new file mode 100644 index 000000000..5f06a063a --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/Position.java @@ -0,0 +1,16 @@ +package org.commonmark.internal.inline; + +/** + * Position within a {@link Scanner}. This is intentionally kept opaque so as not to expose the internal structure of + * the Scanner. + */ +public class Position { + + final int lineIndex; + final int index; + + Position(int lineIndex, int index) { + this.lineIndex = lineIndex; + this.index = index; + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java new file mode 100644 index 000000000..5f6b30f00 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java @@ -0,0 +1,251 @@ +package org.commonmark.internal.inline; + +import org.commonmark.internal.util.CharMatcher; + +import java.util.Collections; +import java.util.List; + +public class Scanner { + + /** + * Character representing the end of input (or outside of the text in case of the "previous" methods). + *

+ * Note that we can use NULL to represent this because CommonMark does not allow those in the input (we replace them + * in the beginning of parsing). + */ + public static final char END = '\0'; + + // Lines without newlines at the end. The scanner will yield `\n` between lines because they're significant for + // parsing and the final output. There is no `\n` after the last line. + private final List lines; + // Which line we're at. + private int lineIndex; + // The index within the line. If index == length(), we pretend that there's a `\n` and only advance after we yield + // that. + private int index; + + // Current line or "" if at the end of the lines (using "" instead of null saves a null check) + private CharSequence line = ""; + private int lineLength = 0; + + Scanner(List lines, int lineIndex, int index) { + this.lines = lines; + this.lineIndex = lineIndex; + this.index = index; + if (!lines.isEmpty()) { + checkPosition(lineIndex, index); + setLine(lines.get(lineIndex)); + } + } + + public static Scanner of(List lines) { + return new Scanner(lines, 0, 0); + } + + public static Scanner of(CharSequence line) { + return new Scanner(Collections.singletonList(line), 0, 0); + } + + public char peek() { + if (index < lineLength) { + return line.charAt(index); + } else { + if (lineIndex < lines.size() - 1) { + return '\n'; + } else { + // Don't return newline for end of last line + return END; + } + } + } + + public char peekPrevious() { + if (index > 0) { + int prev = index - 1; + return line.charAt(prev); + } else { + if (lineIndex > 0) { + return '\n'; + } else { + return END; + } + } + } + + public boolean hasNext() { + if (index < lineLength) { + return true; + } else { + // No newline at end of last line + return lineIndex < lines.size() - 1; + } + } + + public void next() { + index++; + if (index > lineLength) { + lineIndex++; + if (lineIndex < lines.size()) { + setLine(lines.get(lineIndex)); + } else { + setLine(""); + } + index = 0; + } + } + + /** + * Check if the specified char is next and advance the position. + * + * @param c the char to check (including newline characters) + * @return true if matched and position was advanced, false otherwise + */ + public boolean next(char c) { + if (peek() == c) { + next(); + return true; + } else { + return false; + } + } + + /** + * Check if we have the specified content on the line and advanced the position. Note that if you want to match + * newline characters, use {@link #next(char)}. + * + * @param content the text content to match on a single line (excluding newline characters) + * @return true if matched and position was advanced, false otherwise + */ + public boolean next(String content) { + if (index < lineLength && index + content.length() <= lineLength) { + // Can't use startsWith because it's not available on CharSequence + for (int i = 0; i < content.length(); i++) { + if (line.charAt(index + i) != content.charAt(i)) { + return false; + } + } + index += content.length(); + return true; + } else { + return false; + } + } + + public int matchMultiple(char c) { + int count = 0; + while (peek() == c) { + count++; + next(); + } + return count; + } + + public int match(CharMatcher matcher) { + int count = 0; + while (matcher.matches(peek())) { + count++; + next(); + } + return count; + } + + public int whitespace() { + int count = 0; + while (true) { + switch (peek()) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + count++; + next(); + break; + default: + return count; + } + } + } + + public int find(char c) { + int count = 0; + while (true) { + char cur = peek(); + if (cur == Scanner.END) { + return -1; + } else if (cur == c) { + return count; + } + count++; + next(); + } + } + + public int find(CharMatcher matcher) { + int count = 0; + while (true) { + char c = peek(); + if (c == END) { + return -1; + } else if (matcher.matches(c)) { + return count; + } + count++; + next(); + } + } + + // Don't expose the int index, because it would be good if we could switch input to a List of lines later + // instead of one contiguous String. + public Position position() { + return new Position(lineIndex, index); + } + + public void setPosition(Position position) { + checkPosition(position.lineIndex, position.index); + this.lineIndex = position.lineIndex; + this.index = position.index; + setLine(lines.get(this.lineIndex)); + } + + // For cases where the caller appends the result to a StringBuilder, we could offer another method to avoid some + // unnecessary copying. + public CharSequence textBetween(Position begin, Position end) { + if (begin.lineIndex == end.lineIndex) { + // Shortcut for common case of text from a single line + return lines.get(begin.lineIndex).subSequence(begin.index, end.index); + } else { + StringBuilder sb = new StringBuilder(); + + CharSequence firstLine = lines.get(begin.lineIndex); + sb.append(firstLine.subSequence(begin.index, firstLine.length())); + sb.append('\n'); + + // Lines between begin and end (we are appending the full line) + for (int line = begin.lineIndex + 1; line < end.lineIndex; line++) { + sb.append(lines.get(line)); + sb.append('\n'); + } + + CharSequence lastLine = lines.get(end.lineIndex); + sb.append(lastLine.subSequence(0, end.index)); + return sb.toString(); + } + } + + private void setLine(CharSequence line) { + this.line = line; + this.lineLength = line.length(); + } + + private void checkPosition(int lineIndex, int index) { + if (lineIndex < 0 || lineIndex >= lines.size()) { + throw new IllegalArgumentException("Line index " + lineIndex + " out of range, number of lines: " + lines.size()); + } + CharSequence line = lines.get(lineIndex); + if (index < 0 || index > line.length()) { + throw new IllegalArgumentException("Index " + index + " out of range, line length: " + line.length()); + } + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java new file mode 100644 index 000000000..82d83ca46 --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java @@ -0,0 +1,51 @@ +package org.commonmark.internal.util; + +import java.util.BitSet; + +public class AsciiMatcher implements CharMatcher { + private final BitSet set; + + private AsciiMatcher(Builder builder) { + this.set = builder.set; + } + + @Override + public boolean matches(char c) { + return set.get(c); + } + + public Builder newBuilder() { + return new Builder((BitSet) set.clone()); + } + + public static Builder builder() { + return new Builder(new BitSet()); + } + + public static class Builder { + private final BitSet set; + + private Builder(BitSet set) { + this.set = set; + } + + public Builder c(char c) { + if (c > 127) { + throw new IllegalArgumentException("Can only match ASCII characters"); + } + set.set(c); + return this; + } + + public Builder range(char from, char toInclusive) { + for (char c = from; c <= toInclusive; c++) { + c(c); + } + return this; + } + + public AsciiMatcher build() { + return new AsciiMatcher(this); + } + } +} diff --git a/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java new file mode 100644 index 000000000..de730e90d --- /dev/null +++ b/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java @@ -0,0 +1,6 @@ +package org.commonmark.internal.util; + +public interface CharMatcher { + + boolean matches(char c); +} diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Escaping.java b/commonmark/src/main/java/org/commonmark/internal/util/Escaping.java index 15197556c..2b34f6190 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/Escaping.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/Escaping.java @@ -111,12 +111,6 @@ public static String percentEncodeUrl(String s) { return replaceAll(ESCAPE_IN_URI, s, URI_REPLACER); } - public static String normalizeReference(String input) { - // Strip '[' and ']' - String stripped = input.substring(1, input.length() - 1); - return normalizeLabelContent(stripped); - } - public static String normalizeLabelContent(String input) { String trimmed = input.trim(); String lowercase = trimmed.toLowerCase(Locale.ROOT); diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Html5Entities.java b/commonmark/src/main/java/org/commonmark/internal/util/Html5Entities.java index 5215a44df..523c596ed 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/Html5Entities.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/Html5Entities.java @@ -5,24 +5,31 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class Html5Entities { private static final Map NAMED_CHARACTER_REFERENCES = readEntities(); - private static final Pattern NUMERIC_PATTERN = Pattern.compile("^&#[Xx]?"); private static final String ENTITY_PATH = "/org/commonmark/internal/util/entities.properties"; public static String entityToString(String input) { - Matcher matcher = NUMERIC_PATTERN.matcher(input); + if (!input.startsWith("&") || !input.endsWith(";")) { + return input; + } + + String value = input.substring(1, input.length() - 1); + if (value.startsWith("#")) { + value = value.substring(1); + int base = 10; + if (value.startsWith("x") || value.startsWith("X")) { + value = value.substring(1); + base = 16; + } - if (matcher.find()) { - int base = matcher.end() == 2 ? 10 : 16; try { - int codePoint = Integer.parseInt(input.substring(matcher.end(), input.length() - 1), base); + int codePoint = Integer.parseInt(value, base); if (codePoint == 0) { return "\uFFFD"; } @@ -31,8 +38,7 @@ public static String entityToString(String input) { return "\uFFFD"; } } else { - String name = input.substring(1, input.length() - 1); - String s = NAMED_CHARACTER_REFERENCES.get(name); + String s = NAMED_CHARACTER_REFERENCES.get(value); if (s != null) { return s; } else { @@ -44,7 +50,7 @@ public static String entityToString(String input) { private static Map readEntities() { Map entities = new HashMap<>(); InputStream stream = Html5Entities.class.getResourceAsStream(ENTITY_PATH); - Charset charset = Charset.forName("UTF-8"); + Charset charset = StandardCharsets.UTF_8; try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream, charset))) { String line; while ((line = bufferedReader.readLine()) != null) { diff --git a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java index f25cd59e5..3ca34c5f0 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java @@ -1,69 +1,76 @@ package org.commonmark.internal.util; +import org.commonmark.internal.inline.Scanner; + public class LinkScanner { /** - * Attempt to scan the contents of a link label (inside the brackets), returning the position after the content or - * -1. The returned position can either be the closing {@code ]}, or the end of the line if the label continues on + * Attempt to scan the contents of a link label (inside the brackets), stopping after the content or returning false. + * The stopped position can bei either the closing {@code ]}, or the end of the line if the label continues on * the next line. */ - public static int scanLinkLabelContent(CharSequence input, int start) { - for (int i = start; i < input.length(); i++) { - char c = input.charAt(i); - switch (c) { + public static boolean scanLinkLabelContent(Scanner scanner) { + while (scanner.hasNext()) { + switch (scanner.peek()) { case '\\': - if (Parsing.isEscapable(input, i + 1)) { - i += 1; + scanner.next(); + if (Parsing.isEscapable(scanner.peek())) { + scanner.next(); } break; case ']': - return i; + return true; case '[': // spec: Unescaped square bracket characters are not allowed inside the opening and closing // square brackets of link labels. - return -1; + return false; + default: + scanner.next(); } } - return input.length(); + return true; } /** - * Attempt to scan a link destination, returning the position after the destination or -1. + * Attempt to scan a link destination, stopping after the destination or returning false. */ - public static int scanLinkDestination(CharSequence input, int start) { - if (start >= input.length()) { - return -1; + public static boolean scanLinkDestination(Scanner scanner) { + if (!scanner.hasNext()) { + return false; } - if (input.charAt(start) == '<') { - for (int i = start + 1; i < input.length(); i++) { - char c = input.charAt(i); - switch (c) { + if (scanner.next('<')) { + while (scanner.hasNext()) { + switch (scanner.peek()) { case '\\': - if (Parsing.isEscapable(input, i + 1)) { - i += 1; + scanner.next(); + if (Parsing.isEscapable(scanner.peek())) { + scanner.next(); } break; case '\n': case '<': - return -1; + return false; case '>': - return i + 1; + scanner.next(); + return true; + default: + scanner.next(); } } - return -1; + return false; } else { - return scanLinkDestinationWithBalancedParens(input, start); + return scanLinkDestinationWithBalancedParens(scanner); } } - public static int scanLinkTitle(CharSequence input, int start) { - if (start >= input.length()) { - return -1; + public static boolean scanLinkTitle(Scanner scanner) { + if (!scanner.hasNext()) { + return false; } char endDelimiter; - switch (input.charAt(start)) { + switch (scanner.peek()) { case '"': endDelimiter = '"'; break; @@ -74,75 +81,83 @@ public static int scanLinkTitle(CharSequence input, int start) { endDelimiter = ')'; break; default: - return -1; + return false; } + scanner.next(); - int afterContent = scanLinkTitleContent(input, start + 1, endDelimiter); - if (afterContent == -1) { - return -1; + if (!scanLinkTitleContent(scanner, endDelimiter)) { + return false; } - - if (afterContent >= input.length() || input.charAt(afterContent) != endDelimiter) { - // missing or wrong end delimiter - return -1; + if (!scanner.hasNext()) { + return false; } - - return afterContent + 1; + scanner.next(); + return true; } - public static int scanLinkTitleContent(CharSequence input, int start, char endDelimiter) { - for (int i = start; i < input.length(); i++) { - char c = input.charAt(i); - if (c == '\\' && Parsing.isEscapable(input, i + 1)) { - i += 1; + public static boolean scanLinkTitleContent(Scanner scanner, char endDelimiter) { + while (scanner.hasNext()) { + char c = scanner.peek(); + if (c == '\\') { + scanner.next(); + if (Parsing.isEscapable(scanner.peek())) { + scanner.next(); + } } else if (c == endDelimiter) { - return i; + return true; } else if (endDelimiter == ')' && c == '(') { // unescaped '(' in title within parens is invalid - return -1; + return false; + } else { + scanner.next(); } } - return input.length(); + return true; } // spec: a nonempty sequence of characters that does not start with <, does not include ASCII space or control // characters, and includes parentheses only if (a) they are backslash-escaped or (b) they are part of a balanced // pair of unescaped parentheses - private static int scanLinkDestinationWithBalancedParens(CharSequence input, int start) { + private static boolean scanLinkDestinationWithBalancedParens(Scanner scanner) { int parens = 0; - for (int i = start; i < input.length(); i++) { - char c = input.charAt(i); + boolean empty = true; + while (scanner.hasNext()) { + char c = scanner.peek(); switch (c) { - case '\0': case ' ': - return i != start ? i : -1; + return !empty; case '\\': - if (Parsing.isEscapable(input, i + 1)) { - i += 1; + scanner.next(); + if (Parsing.isEscapable(scanner.peek())) { + scanner.next(); } break; case '(': parens++; // Limit to 32 nested parens for pathological cases if (parens > 32) { - return -1; + return false; } + scanner.next(); break; case ')': if (parens == 0) { - return i; + return true; } else { parens--; } + scanner.next(); break; default: // or control character if (Character.isISOControl(c)) { - return i != start ? i : -1; + return !empty; } + scanner.next(); break; } + empty = false; } - return input.length(); + return true; } } diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java index d429d9db0..eb7dddb0d 100644 --- a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java +++ b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java @@ -72,43 +72,41 @@ public static boolean isSpaceOrTab(CharSequence s, int index) { return false; } - public static boolean isEscapable(CharSequence s, int index) { - if (index < s.length()) { - switch (s.charAt(index)) { - case '!': - case '"': - case '#': - case '$': - case '%': - case '&': - case '\'': - case '(': - case ')': - case '*': - case '+': - case ',': - case '-': - case '.': - case '/': - case ':': - case ';': - case '<': - case '=': - case '>': - case '?': - case '@': - case '[': - case '\\': - case ']': - case '^': - case '_': - case '`': - case '{': - case '|': - case '}': - case '~': - return true; - } + public static boolean isEscapable(char c) { + switch (c) { + case '!': + case '"': + case '#': + case '$': + case '%': + case '&': + case '\'': + case '(': + case ')': + case '*': + case '+': + case ',': + case '-': + case '.': + case '/': + case ':': + case ';': + case '<': + case '=': + case '>': + case '?': + case '@': + case '[': + case '\\': + case ']': + case '^': + case '_': + case '`': + case '{': + case '|': + case '}': + case '~': + return true; } return false; } @@ -122,18 +120,16 @@ public static CharSequence prepareLine(CharSequence line) { int length = line.length(); for (int i = 0; i < length; i++) { char c = line.charAt(i); - switch (c) { - case '\0': - if (sb == null) { - sb = new StringBuilder(length); - sb.append(line, 0, i); - } - sb.append('\uFFFD'); - break; - default: - if (sb != null) { - sb.append(c); - } + if (c == '\0') { + if (sb == null) { + sb = new StringBuilder(length); + sb.append(line, 0, i); + } + sb.append('\uFFFD'); + } else { + if (sb != null) { + sb.append(c); + } } } @@ -188,6 +184,23 @@ public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int last return lastIndex - 1; } + public static int skipWhitespace(CharSequence s, int startIndex, int endIndex) { + for (int i = startIndex; i < endIndex; i++) { + switch (s.charAt(i)) { + case ' ': + case '\t': + case '\n': + case '\u000B': + case '\f': + case '\r': + break; + default: + return i; + } + } + return endIndex; + } + private static int findNonSpace(CharSequence s, int startIndex) { int length = s.length(); for (int i = startIndex; i < length; i++) { diff --git a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java index 492c3cc8a..cbebdc25a 100644 --- a/commonmark/src/main/java/org/commonmark/parser/InlineParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/InlineParser.java @@ -2,14 +2,16 @@ import org.commonmark.node.Node; +import java.util.List; + /** * Parser for inline content (text, links, emphasized text, etc). */ public interface InlineParser { /** - * @param input the content to parse as inline + * @param lines the content to parse as inline * @param node the node to append resulting nodes to (as children) */ - void parse(String input, Node node); + void parse(List lines, Node node); } diff --git a/commonmark/src/main/java/org/commonmark/parser/block/MatchedBlockParser.java b/commonmark/src/main/java/org/commonmark/parser/block/MatchedBlockParser.java index d4cd9d471..41ba2712b 100644 --- a/commonmark/src/main/java/org/commonmark/parser/block/MatchedBlockParser.java +++ b/commonmark/src/main/java/org/commonmark/parser/block/MatchedBlockParser.java @@ -1,5 +1,7 @@ package org.commonmark.parser.block; +import java.util.List; + /** * Open block parser that was last matched during the continue phase. This is different from the currently active * block parser, as an unmatched block is only closed when a new block is started. @@ -10,11 +12,10 @@ public interface MatchedBlockParser { BlockParser getMatchedBlockParser(); /** - * Returns the current content of the paragraph if the matched block is a paragraph. The content can be multiple - * lines separated by {@code '\n'}. + * Returns the current paragraph lines if the matched block is a paragraph. * - * @return paragraph content or {@code null} + * @return paragraph content or an empty list */ - CharSequence getParagraphContent(); + List getParagraphLines(); } diff --git a/commonmark/src/test/java/org/commonmark/internal/LinkReferenceDefinitionParserTest.java b/commonmark/src/test/java/org/commonmark/internal/LinkReferenceDefinitionParserTest.java index f0bdef492..3b9ae83dc 100644 --- a/commonmark/src/test/java/org/commonmark/internal/LinkReferenceDefinitionParserTest.java +++ b/commonmark/src/test/java/org/commonmark/internal/LinkReferenceDefinitionParserTest.java @@ -12,9 +12,7 @@ public class LinkReferenceDefinitionParserTest { @Test public void testStartLabel() { - parser.parse("["); - assertEquals(State.LABEL, parser.getState()); - assertEquals("[", parser.getParagraphContent().toString()); + assertState("[", State.LABEL, "["); } @Test @@ -25,7 +23,7 @@ public void testStartNoLabel() { parser.parse("a"); parser.parse("["); assertEquals(State.PARAGRAPH, parser.getState()); - assertEquals("a\n[", parser.getParagraphContent().toString()); + assertParagraphLines("a\n[", parser); } @Test @@ -80,7 +78,7 @@ public void testDestination() { LinkReferenceDefinitionParser parser = new LinkReferenceDefinitionParser(); parser.parse("[foo]: /url"); assertEquals(State.START_TITLE, parser.getState()); - assertEquals("", parser.getParagraphContent().toString()); + assertParagraphLines("", parser); assertEquals(1, parser.getDefinitions().size()); assertDef(parser.getDefinitions().get(0), "foo", "/url", null); @@ -99,7 +97,7 @@ public void testTitle() { LinkReferenceDefinitionParser parser = new LinkReferenceDefinitionParser(); parser.parse("[foo]: /url 'title'"); assertEquals(State.START_DEFINITION, parser.getState()); - assertEquals("", parser.getParagraphContent().toString()); + assertParagraphLines("", parser); assertEquals(1, parser.getDefinitions().size()); assertDef(parser.getDefinitions().get(0), "foo", "/url", "title"); @@ -110,12 +108,12 @@ public void testTitleStartWhitespace() { LinkReferenceDefinitionParser parser = new LinkReferenceDefinitionParser(); parser.parse("[foo]: /url"); assertEquals(State.START_TITLE, parser.getState()); - assertEquals("", parser.getParagraphContent().toString()); + assertParagraphLines("", parser); parser.parse(" "); assertEquals(State.START_DEFINITION, parser.getState()); - assertEquals(" ", parser.getParagraphContent().toString()); + assertParagraphLines(" ", parser); assertEquals(1, parser.getDefinitions().size()); assertDef(parser.getDefinitions().get(0), "foo", "/url", null); @@ -126,17 +124,17 @@ public void testTitleMultiline() { LinkReferenceDefinitionParser parser = new LinkReferenceDefinitionParser(); parser.parse("[foo]: /url 'two"); assertEquals(State.TITLE, parser.getState()); - assertEquals("[foo]: /url 'two", parser.getParagraphContent().toString()); + assertParagraphLines("[foo]: /url 'two", parser); assertEquals(0, parser.getDefinitions().size()); parser.parse("lines"); assertEquals(State.TITLE, parser.getState()); - assertEquals("[foo]: /url 'two\nlines", parser.getParagraphContent().toString()); + assertParagraphLines("[foo]: /url 'two\nlines", parser); assertEquals(0, parser.getDefinitions().size()); parser.parse("'"); assertEquals(State.START_DEFINITION, parser.getState()); - assertEquals("", parser.getParagraphContent().toString()); + assertParagraphLines("", parser); assertEquals(1, parser.getDefinitions().size()); assertDef(parser.getDefinitions().get(0), "foo", "/url", "two\nlines\n"); @@ -168,7 +166,7 @@ private static void assertState(String input, State state, String paragraphConte LinkReferenceDefinitionParser parser = new LinkReferenceDefinitionParser(); parser.parse(input); assertEquals(state, parser.getState()); - assertEquals(paragraphContent, parser.getParagraphContent().toString()); + assertParagraphLines(paragraphContent, parser); } private static void assertDef(LinkReferenceDefinition def, String label, String destination, String title) { @@ -176,4 +174,16 @@ private static void assertDef(LinkReferenceDefinition def, String label, String assertEquals(destination, def.getDestination()); assertEquals(title, def.getTitle()); } + + private static void assertParagraphLines(String expectedContent, LinkReferenceDefinitionParser parser) { + StringBuilder sb = new StringBuilder(); + for (CharSequence line : parser.getParagraphLines()) { + if (sb.length() != 0) { + sb.append('\n'); + } + sb.append(line); + } + String actual = sb.toString(); + assertEquals(expectedContent, actual); + } } diff --git a/commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java b/commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java new file mode 100644 index 000000000..c68140a71 --- /dev/null +++ b/commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java @@ -0,0 +1,93 @@ +package org.commonmark.internal.inline; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.*; + +public class ScannerTest { + + @Test + public void testNext() { + Scanner scanner = new Scanner(Collections.singletonList("foo bar"), 0, 4); + assertEquals('b', scanner.peek()); + scanner.next(); + assertEquals('a', scanner.peek()); + scanner.next(); + assertEquals('r', scanner.peek()); + scanner.next(); + assertEquals('\0', scanner.peek()); + } + + @Test + public void testMultipleLines() { + Scanner scanner = new Scanner(Arrays.asList("ab", "cde"), 0, 0); + assertTrue(scanner.hasNext()); + assertEquals('\0', scanner.peekPrevious()); + assertEquals('a', scanner.peek()); + scanner.next(); + + assertTrue(scanner.hasNext()); + assertEquals('a', scanner.peekPrevious()); + assertEquals('b', scanner.peek()); + scanner.next(); + + assertTrue(scanner.hasNext()); + assertEquals('b', scanner.peekPrevious()); + assertEquals('\n', scanner.peek()); + scanner.next(); + + assertTrue(scanner.hasNext()); + assertEquals('\n', scanner.peekPrevious()); + assertEquals('c', scanner.peek()); + scanner.next(); + + assertTrue(scanner.hasNext()); + assertEquals('c', scanner.peekPrevious()); + assertEquals('d', scanner.peek()); + scanner.next(); + + assertTrue(scanner.hasNext()); + assertEquals('d', scanner.peekPrevious()); + assertEquals('e', scanner.peek()); + scanner.next(); + + assertFalse(scanner.hasNext()); + assertEquals('e', scanner.peekPrevious()); + assertEquals('\0', scanner.peek()); + } + + @Test + public void testTextBetween() { + Scanner scanner = new Scanner(Arrays.asList("ab", "cde"), 0, 0); + Position start = scanner.position(); + scanner.next(); + assertEquals("a", scanner.textBetween(start, scanner.position())); + Position afterA = scanner.position(); + scanner.next(); + assertEquals("ab", scanner.textBetween(start, scanner.position())); + scanner.next(); + assertEquals("ab\n", scanner.textBetween(start, scanner.position())); + scanner.next(); + assertEquals("ab\nc", scanner.textBetween(start, scanner.position())); + scanner.next(); + assertEquals("ab\ncd", scanner.textBetween(start, scanner.position())); + scanner.next(); + assertEquals("ab\ncde", scanner.textBetween(start, scanner.position())); + + assertEquals("b\ncde", scanner.textBetween(afterA, scanner.position())); + } + + @Test + public void nextString() { + Scanner scanner = Scanner.of(Arrays.asList("hey ya", "hi")); + assertFalse(scanner.next("hoy")); + assertTrue(scanner.next("hey")); + assertTrue(scanner.next(' ')); + assertFalse(scanner.next("yo")); + assertTrue(scanner.next("ya")); + assertFalse(scanner.next(" ")); + } +} diff --git a/commonmark/src/test/java/org/commonmark/test/HtmlInlineParserTest.java b/commonmark/src/test/java/org/commonmark/test/HtmlInlineParserTest.java new file mode 100644 index 000000000..0a406778b --- /dev/null +++ b/commonmark/src/test/java/org/commonmark/test/HtmlInlineParserTest.java @@ -0,0 +1,27 @@ +package org.commonmark.test; + +import org.junit.Test; + +public class HtmlInlineParserTest extends CoreRenderingTestCase { + + @Test + public void comment() { + assertRendering("inline ", "

inline

\n"); + assertRendering("inline ", "

inline

\n"); + assertRendering("inline -->", "

inline <!--->-->

\n"); + } + + @Test + public void cdata() { + assertRendering("inline ", "

inline

\n"); + assertRendering("inline ", "

inline

\n"); + } + + @Test + public void declaration() { + // Whitespace is mandatory + assertRendering("inline ", "

inline <!FOO>

\n"); + assertRendering("inline ", "

inline

\n"); + assertRendering("inline ", "

inline

\n"); + } +} diff --git a/commonmark/src/test/java/org/commonmark/test/ParserTest.java b/commonmark/src/test/java/org/commonmark/test/ParserTest.java index e058de378..216bdeefe 100644 --- a/commonmark/src/test/java/org/commonmark/test/ParserTest.java +++ b/commonmark/src/test/java/org/commonmark/test/ParserTest.java @@ -103,7 +103,7 @@ public void indentation() { public void inlineParser() { final InlineParser fakeInlineParser = new InlineParser() { @Override - public void parse(String input, Node node) { + public void parse(List lines, Node node) { node.appendChild(new ThematicBreak()); } }; diff --git a/commonmark/src/test/java/org/commonmark/test/SpecialInputTest.java b/commonmark/src/test/java/org/commonmark/test/SpecialInputTest.java index a70127a72..2b19db3db 100644 --- a/commonmark/src/test/java/org/commonmark/test/SpecialInputTest.java +++ b/commonmark/src/test/java/org/commonmark/test/SpecialInputTest.java @@ -167,4 +167,11 @@ public void deeplyIndentedList() { "\n" + "\n"); } + + @Test + public void trailingTabs() { + // The tab is not treated as 4 spaces here and so does not result in a hard line break, but is just preserved. + // This matches what commonmark.js did at the time of writing. + assertRendering("a\t\nb\n", "

a\t\nb

\n"); + } }