From 85f4859e268ad4b02cb2c591fd02ac667d70541b Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Fri, 1 May 2026 08:01:42 +0200 Subject: [PATCH 1/3] feat: Markdown semantic tokens This PR adds semantic tokens for Markdown and Verso that correspond to those available in editor themes for Markdown. With a small update to editors, docstrings can be consistently formatted without needing to reimplement either complex parser client-side. --- src/Lean/Data/Lsp/LanguageFeatures.lean | 48 +- src/Lean/Server/FileWorker.lean | 1 + src/Lean/Server/FileWorker/Markdown.lean | 23 + .../Server/FileWorker/Markdown/Basic.lean | 334 + .../FileWorker/Markdown/InlineSource.lean | 241 + .../Server/FileWorker/Markdown/Parser.lean | 57 + .../FileWorker/Markdown/Parser/Block.lean | 1194 ++ .../Markdown/Parser/BlockZipper.lean | 253 + .../FileWorker/Markdown/Parser/Cursor.lean | 96 + .../FileWorker/Markdown/Parser/Emphasis.lean | 286 + .../FileWorker/Markdown/Parser/Inline.lean | 715 ++ .../FileWorker/SemanticHighlighting.lean | 328 +- src/Lean/Server/ProtocolOverview.lean | 6 +- tests/CMakeLists.txt | 1 + tests/elab/markdown_token_merge.lean | 253 + tests/markdown_conformance/README.txt | 10 + .../conformance.out.expected | 87 + tests/markdown_conformance/run_test.sh | 9 + tests/markdown_conformance/runner.lean | 776 ++ tests/markdown_conformance/spec.txt | 9811 +++++++++++++++++ .../semanticTokensMarkdownDocs.lean | 103 + ...manticTokensMarkdownDocs.lean.out.expected | 1712 +++ .../semanticTokensVersoDocs.lean | 24 +- .../semanticTokensVersoDocs.lean.out.expected | 436 +- 24 files changed, 16745 insertions(+), 59 deletions(-) create mode 100644 src/Lean/Server/FileWorker/Markdown.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Basic.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/InlineSource.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser/Block.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser/BlockZipper.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser/Cursor.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser/Emphasis.lean create mode 100644 src/Lean/Server/FileWorker/Markdown/Parser/Inline.lean create mode 100644 tests/elab/markdown_token_merge.lean create mode 100644 tests/markdown_conformance/README.txt create mode 100644 tests/markdown_conformance/conformance.out.expected create mode 100644 tests/markdown_conformance/run_test.sh create mode 100644 tests/markdown_conformance/runner.lean create mode 100644 tests/markdown_conformance/spec.txt create mode 100644 tests/server_interactive/semanticTokensMarkdownDocs.lean create mode 100644 tests/server_interactive/semanticTokensMarkdownDocs.lean.out.expected diff --git a/src/Lean/Data/Lsp/LanguageFeatures.lean b/src/Lean/Data/Lsp/LanguageFeatures.lean index 548df6b0e35f..fad37fcda7a0 100644 --- a/src/Lean/Data/Lsp/LanguageFeatures.lean +++ b/src/Lean/Data/Lsp/LanguageFeatures.lean @@ -454,6 +454,46 @@ inductive SemanticTokenType where | decorator -- Extensions | leanSorryLike + -- Markdown/Verso markup highlighting. The composing types combine an + -- emphasis subset (none, bold, italic, or both) with a block context (none, + -- heading, blockquote, list). + | markupBold + | markupItalic + | markupBoldItalic + | markupHeading + | markupBoldHeading + | markupItalicHeading + | markupBoldItalicHeading + | markupQuote + | markupBoldQuote + | markupItalicQuote + | markupBoldItalicQuote + | markupList + | markupBoldList + | markupItalicList + | markupBoldItalicList + | markupInlineCode + | markupCodeBlock + /-- + Plain markup text inside a docstring or moduledoc. This is used for Verso or Markdown content that + is not otherwise styled, allowing themes to apply the correct style. + -/ + | markupDocText + /-- + Plain markup text outside of a docstring or moduledoc. This is used for Verso or Markdown content + that is not otherwise styled in contexts that are not docstrings (such as full-document Verso + content), allowing themes to style it the way they would ordinary text content. + -/ + | markupPlainText + /-- + A URL literal in an inline link/image target or autolink. + -/ + | markupUrl + /-- + A cross-reference label: the `[label]` of a link reference definition, or the label part of a + reference-style or shortcut link. + -/ + | markupCrossReference deriving ToJson, FromJson, BEq, Hashable -- must be in the same order as the constructors @@ -461,7 +501,13 @@ def SemanticTokenType.names : Array String := #["keyword", "variable", "property", "function", "namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "enumMember", "event", "method", "macro", "modifier", "comment", "string", "number", - "regexp", "operator", "decorator", "leanSorryLike"] + "regexp", "operator", "decorator", "leanSorryLike", + "markupBold", "markupItalic", "markupBoldItalic", + "markupHeading", "markupBoldHeading", "markupItalicHeading", "markupBoldItalicHeading", + "markupQuote", "markupBoldQuote", "markupItalicQuote", "markupBoldItalicQuote", + "markupList", "markupBoldList", "markupItalicList", "markupBoldItalicList", + "markupInlineCode", "markupCodeBlock", + "markupDocText", "markupPlainText", "markupUrl", "markupCrossReference"] def SemanticTokenType.toNat (tokenType : SemanticTokenType) : Nat := tokenType.ctorIdx diff --git a/src/Lean/Server/FileWorker.lean b/src/Lean/Server/FileWorker.lean index c7fd7c8dbd97..bb15a6006ce3 100644 --- a/src/Lean/Server/FileWorker.lean +++ b/src/Lean/Server/FileWorker.lean @@ -18,6 +18,7 @@ public import Lean.Server.FileWorker.Utils public import Lean.Server.FileWorker.RequestHandling public import Lean.Server.FileWorker.WidgetRequests public import Lean.Server.FileWorker.SetupFile +public import Lean.Server.FileWorker.Markdown public import Lean.Server.Completion.ImportCompletion public import Lean.Server.CodeActions.UnknownIdentifier diff --git a/src/Lean/Server/FileWorker/Markdown.lean b/src/Lean/Server/FileWorker/Markdown.lean new file mode 100644 index 000000000000..d2bd3a916aac --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown.lean @@ -0,0 +1,23 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Lean.Server.FileWorker.Markdown.Basic +public import Lean.Server.FileWorker.Markdown.Parser + +/-! +# Markdown parser for the language server + +A CommonMark parser used by the Lean language server to provide semantic token highlighting inside +Markdown docstrings. The entry point is `Lean.Server.FileWorker.Markdown.parseDocument`. + +The implementation follows CommonMark 0.31.2 closely with a small set of deviations: +* HTML blocks and inline HTML are not parsed. +* Link labels are compared case-insensitively only for the English letters A-Z, and case-sensitively + for all other characters. +* Named character entities are not validated against the set of HTML entities. +-/ diff --git a/src/Lean/Server/FileWorker/Markdown/Basic.lean b/src/Lean/Server/FileWorker/Markdown/Basic.lean new file mode 100644 index 000000000000..59999eef99dc --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Basic.lean @@ -0,0 +1,334 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Init.Data.Hashable +public import Std.Data.HashMap +public import Lean.Syntax + +public section + +namespace Lean.Server.FileWorker.Markdown + +/-- The marker character of a bullet list. -/ +inductive BulletKind where + /-- A `*`-marked bullet list. -/ + | star + /-- A `-`-marked bullet list. -/ + | hyphen + /-- A `+`-marked bullet list. -/ + | plus +deriving Repr, BEq, Hashable + +/-- The terminating delimiter of an ordered-list marker. -/ +inductive OrderedDelim where + /-- A `.` delimiter, as in `1.` -/ + | dot + /-- A `)` delimiter, as in `1)` -/ + | rparen +deriving Repr, BEq, Hashable + +/-- +The kind of a list. All items in a list must share the same kind: an unordered list cannot mix `*` +with `-`, and an ordered list cannot mix `1.` with `1)`. +-/ +inductive ListKind where + /-- A bullet list with the given marker character. -/ + | bullet (kind : BulletKind) + /-- + An ordered list. `start` is the value of the first item; `delim` is the + punctuation following the digits. + -/ + | ordered (start : Nat) (delim : OrderedDelim) +deriving Repr, BEq, Hashable + +/-- The fence character used by a fenced code block. -/ +inductive FenceChar where + /-- A backtick (`` ` ``) fence. The info string may not contain backticks. -/ + | backtick + /-- A tilde (`~`) fence. -/ + | tilde +deriving Repr, BEq, Hashable, Inhabited + +namespace FenceChar + +def ofChar? : Char → Option FenceChar + | '`' => some .backtick + | '~' => some .tilde + | _ => none + +def toChar : FenceChar → Char + | .backtick => '`' + | .tilde => '~' + +end FenceChar + +/-- Information attached to a fenced code block at its opening fence. -/ +structure FenceInfo where + /-- The fence character: `` ` `` or `~`. -/ + fenceChar : FenceChar + /-- + The length of the opening fence run (always 3 or more). The closing fence must be at least this + long and use the same character. + -/ + fenceLen : Nat + /-- + Indentation in spaces of the opening fence's first character. Up to this many leading spaces are + stripped from each content line before it is treated as code. + -/ + openIndent : Nat + /-- The info string range, if it exists. This is usually a language tag. -/ + infoString? : Option Syntax.Range +deriving Repr, BEq, Hashable, Inhabited + +/-- +The source ranges of a link reference definition's structural parts. The +label is the bracket interior (excluding the brackets themselves); the +URL is the destination (including any `<>` wrappers if used); `title?` +is the optional title's interior (excluding the surrounding quote/paren +delimiters). + +For multi-line reference definitions, ranges may span newlines. +-/ +structure RefDef where + /-- The opening `[` of the label. -/ + openBracket : Syntax.Range + /-- The label's interior, between the brackets (un-normalized). -/ + label : Syntax.Range + /-- The closing `]` of the label. -/ + closeBracket : Syntax.Range + /-- The `:` separating the label from the destination. -/ + colon : Syntax.Range + /-- + The destination's source range. Includes the `<>` wrappers if the destination uses bracketed form. + -/ + url : Syntax.Range + /-- + The optional title's interior, excluding its surrounding delimiters (`"…"`, `'…'`, or `(…)`). + -/ + title? : Option Syntax.Range +deriving Repr, Inhabited + +/-- +A fully parsed block-level subtree. + +The type parameter `α` is the kind of inline-bearing content each block holds: + +- After the first pass, in which the type and extent of all blocks have been determined but the + inlines have not yet been parsed, `α := Syntax.Range`. Each paragraph or heading line is one such + range. +- After the second pass, which parses the inline elements, `α := Array Inline`. The resulting + `lines` array of a `paragraph` or `setextHeading` always contains exactly one element: the inline + element syntax tree that represents the whole paragraph. + +Verbatim blocks (`fencedCode`, `indentedCode`) and structural blocks (`linkRefDef`) do not carry +`α`-typed content; they only ever hold raw source ranges, regardless of which pass has run. +-/ +inductive Block (α : Type) where + /-- + A paragraph: a sequence of consecutive non-blank lines that did not open + any other leaf. + -/ + | paragraph (lines : Array α) + /-- + An ATX heading (the usual kind). `hashes` covers the opening run of `#` characters, + `closeHashes?` covers an optional closing run. + -/ + | atxHeading (hashes : Syntax.Range) (content : α) (closeHashes? : Option Syntax.Range) + /-- + A setext heading (CommonMark §4.3): one or more lines of paragraph text followed by an underline + (`===` for level 1, `---` for level 2). `lines` is the content of the preceding paragraph, and + `underline` covers the underline itself. + -/ + | setextHeading (level : Nat) (lines : Array α) (underline : Syntax.Range) + /-- + A fenced code block. `openFence`/`closeFence?` cover the fences runs themselves. The closing fence + is optional because an unterminated fenced code block spans the rest of the input. + -/ + | fencedCode (info : FenceInfo) + (openFence : Syntax.Range) (closeFence? : Option Syntax.Range) + (lines : Array Syntax.Range) + /-- + An indented code block: a sequence of lines indented four or more columns. Each element is a + pair of the line's source range (covering the whole source line, including the indent that the + block consumed) and its processed content. In the processed content, the leading 4 cols of indent + (relative to the parent container) have been stripped, with any partial tabs materialized as the + appropriate number of spaces per CommonMark §2.2. + -/ + | indentedCode (lines : Array (Syntax.Range × String)) + /-- + A blockquote container. `markers` records the position of every `>` marker located during parsing, + in source order and `children` holds the contained blocks. + -/ + | blockquote (markers : Array Syntax.Range) (children : Array (Block α)) + /-- + A list container. All `items` are `listItem` blocks. `tight` reflects CommonMark's tight/loose + distinction (§5.3): a tight list has no blank lines between its items or between blocks within an + item. HTML rendering unwraps `

` from a tight list item's direct paragraph children. + -/ + | list (kind : ListKind) (tight : Bool) (items : Array (Block α)) + /-- + A list item. `marker` is the bullet or ordered-marker range emitted as a delimiter token, and + `children` are the blocks contained inside the item. + -/ + | listItem (marker : Syntax.Range) (children : Array (Block α)) + /-- + A link reference definition: `[label]: destination "title"` (the title is optional). + -/ + | linkRefDef (m : RefDef) + /-- + A thematic break (`***`, `---`, or `___`, optionally with interior spaces or + tabs). The line range covers the entire source line that produced the break. + -/ + | thematicBreak (line : Syntax.Range) +deriving Inhabited + +namespace ListKind + +/-- +Whether two list kinds belong to the same list (that is, they have matching marker styles). + +The starting number of an ordered list is irrelevant: `1.` and `5.` are the same list kind. Bullets +must use the exact same character, and ordered lists must use the same delimiter. +-/ +def sameKind : ListKind → ListKind → Bool + | .bullet a, .bullet b => a == b + | .ordered _ d1, .ordered _ d2 => d1 == d2 + | _, _ => false + +end ListKind + +/-- +The surface form of a reference-style link or image. +-/ +inductive ReferenceLinkForm where + /-- + Full form `[text][label]` — the second brackets carry an explicit + label, distinct from the link's text. `openBracket`/`closeBracket` + cover the second bracket pair; `label` is the bracket interior. + -/ + | full (openBracket : Syntax.Range) (label : Syntax.Range) (closeBracket : Syntax.Range) + /-- + Collapsed form `[text][]`. The second brackets are empty, and the link's text serves as the label. + `openBracket`/`closeBracket` cover the empty second bracket pair. + -/ + | collapsed (openBracket : Syntax.Range) (closeBracket : Syntax.Range) + /-- + Shortcut form `[label]`. There are no second brackets and the link's text serves as the label. + -/ + | shortcut +deriving Repr, Inhabited + +/-- The destination of a link or image. -/ +inductive LinkTarget where + /-- + An inline target `(url)` or `(url "title")`. The title (if present) excludes its surrounding + delimiter characters. + -/ + | inline (openParen : Syntax.Range) (url : Syntax.Range) (closeParen : Syntax.Range) + (title? : Option Syntax.Range := none) + /-- + A reference target. + + The reference table has been consulted at parse time, so `url` and `title?` are already the + resolved destination/title from the matching link reference definition. In Markdown, unresolvable + references never produce a `link`/`image`; instead, the surrounding bracket characters fall + through to plain text. + -/ + | reference (url : String) (title? : Option String) (form : ReferenceLinkForm) +deriving Repr, Inhabited + +/-- A parsed inline element. -/ +inductive Inline where + /-- Plain text. -/ + | text (range : Syntax.Range) + /-- Inline code span. The content is verbatim source. -/ + | code (openTicks : Syntax.Range) (content : Syntax.Range) (closeTicks : Syntax.Range) + /-- Italic emphasis. -/ + | italic (openDelim : Syntax.Range) (content : Array Inline) (closeDelim : Syntax.Range) + /-- Bold/strong emphasis. -/ + | bold (openDelim : Syntax.Range) (content : Array Inline) (closeDelim : Syntax.Range) + /-- Inline or reference link. The text is recursively parsed. -/ + | link (openBracket : Syntax.Range) (text : Array Inline) (closeBracket : Syntax.Range) + (target : LinkTarget) + /-- + An image. The alt content is recursively parsed as inline content (CommonMark §6.4); HTML + rendering should flatten it to plain text for the `alt` attribute. + -/ + | image (bang : Syntax.Range) (openBracket : Syntax.Range) (alt : Array Inline) + (closeBracket : Syntax.Range) (target : LinkTarget) + /-- + A hard line break. These are produced by a backslash immediately before a line ending or by two or + more trailing spaces before a line ending (CommonMark §6.7). The range covers the trigger + characters (e.g. the `\` for backslash form). + -/ + | hardBreak (range : Syntax.Range) + /-- + An autolink: `` or `` for URI form, or `` for email form + (CommonMark §6.5). The URL between the angle brackets is verbatim — no backslash-escape, entity, + or other markdown processing applies inside. `isEmail` distinguishes the email form, whose href + gets a `mailto:` prefix when rendered to HTML. + -/ + | autolink (openAngle : Syntax.Range) (url : Syntax.Range) (closeAngle : Syntax.Range) + (isEmail : Bool) +deriving Inhabited + +/-- +A normalized link label. + +According to the CommonMark spec §4.7, link labels are normalized as follows: +* They are case-folded (though this implementation does not implement full Unicode case folding) +* Internal whitespace is collapsed to a single space +* Leading and trailing whitespace are removed + +The constructor is private. Build values via `normalizeLabel`. +-/ +structure LinkLabel where + private mk :: + /-- The normalized label string (see `normalizeLabel`). -/ + name : String +deriving BEq, Hashable, Repr + +/-- +Normalizes a CommonMark link label (§4.7): + * Each character is ASCII-lowercased (with `Char.toLower`). + * Internal whitespace runs become single spaces. + * Leading and trailing whitespace is removed. + +While CommonMark specifies Unicode case folding, this implementation case-folds only ASCII English +letters. +-/ +public def normalizeLabel (s : String) : LinkLabel := Id.run do + let mut out := "" + let mut sawWs := false -- whether we've seen whitespace since the last non-ws + let mut sawNonWs := false -- whether we've emitted any non-ws char yet + for c in s do + if c == ' ' || c == '\t' || c == '\n' || c == '\r' then + sawWs := true + else + if sawNonWs && sawWs then out := out.push ' ' + out := out.push c.toLower + sawWs := false + sawNonWs := true + return ⟨out⟩ + +/-- +The destination of a link reference definition: the URL plus optional title. +-/ +structure RefTarget where + /-- The destination URL with any `<>` wrappers stripped. -/ + url : String + /-- The optional title, with its surrounding delimiters stripped. -/ + title? : Option String +deriving Repr, Inhabited + +/-- +A reference table built from a document's link reference definitions. + +During inline parsing, these are used to recognize valid references. +-/ +abbrev RefTable := Std.HashMap LinkLabel RefTarget diff --git a/src/Lean/Server/FileWorker/Markdown/InlineSource.lean b/src/Lean/Server/FileWorker/Markdown/InlineSource.lean new file mode 100644 index 000000000000..8d00c20ffd6b --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/InlineSource.lean @@ -0,0 +1,241 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Init.Data.String.PosRaw +public import Init.Data.Array.Basic +public import Init.While +public import Lean.Syntax + +public section + +namespace Lean.Server.FileWorker.Markdown + +/-! +# Inline Source View + +The block parser carves a paragraph (or heading) into a sequence of source ranges. There is one per +line, with leading container prefixes already stripped. The inline parser then needs to walk that +content as if it were a single contiguous string, while still emitting `Syntax.Range`s anchored in +the *original* file. `InlineSource` it bundles the underlying file string with the per-line ranges, +and exposes `get`/`next` operations that step through the lines while skipping the inter-line gaps +that contain block markers and stripped indentation. +-/ + +/-- +A view onto a `String` source that exposes a logical concatenation of sub-ranges. Each range is one +paragraph line (or analogous content unit) with block markers such as `>` and leading whitespace +stripped from its `start`. The `\n` after each non-final line is the range's last byte (`stop +- 1`), so walking via `get`/`next` sees real `\n` characters at line boundaries. + +Ranges should be non-overlapping. + +Positions returned by walking are file offsets in `str`, so inlines produced over an `InlineSource` +are file-anchored without a separate mapping pass. +-/ +structure InlineSource where + /-- The underlying source string. -/ + str : String + /-- + Per-line ranges in source order. Each `range.start` points past the leading spaces/tabs of its + line, and each `range.stop` points one past the line's terminating `'\n'`, so the `'\n'` is the + range's last byte. The trailing range, if its line had no `'\n'`, ends at the document's EOF. + -/ + ranges : Array Syntax.Range +deriving Inhabited + +namespace InlineSource + +/-- The empty source. -/ +def empty : InlineSource := { str := "", ranges := #[] } + +/-- The first source position covered by `s` (or `0` if empty). -/ +def startPos (s : InlineSource) : String.Pos.Raw := + match s.ranges[0]? with + | some r => r.start + | none => 0 + +/-- One past the last source position covered by `s` (or `0` if empty). -/ +def stopPos (s : InlineSource) : String.Pos.Raw := + match s.ranges.back? with + | some r => r.stop + | none => 0 + +/-- Whether `p` is at or past the end of the last range. -/ +def atEnd (s : InlineSource) (p : String.Pos.Raw) : Bool := + p ≥ s.stopPos + +/-- +Returns the index of the range that contains `p`, or `none` if `p` falls outside every range. +-/ +private def findRangeIdx? (s : InlineSource) (p : String.Pos.Raw) : Option Nat := + s.ranges.findIdx? fun r => + r.start ≤ p && p < r.stop + +/-- +Returns the character at `p`. + +Inside a valid range of `s`, this is simply the character; outside the ranges, it returns +`(default : Char)`. +-/ +def get (s : InlineSource) (p : String.Pos.Raw) : Char := + match s.findRangeIdx? p with + | some _ => p.get s.str + | none => default + +/-- +Returns the next logical position after `p`. Within a range, advances by the UTF-8 width of the +character at `p`. When that advance would land at the end of `p`'s range, it jumps to the start of a +next range if one exists, or to the end of the current range if not. If `p` is not in any range, it +returns `s.stopPos`. +-/ +def next (s : InlineSource) (p : String.Pos.Raw) : String.Pos.Raw := + match s.findRangeIdx? p with + | none => s.stopPos + | some i => + let r := s.ranges[i]! + let next := p + (p.get s.str) + if next < r.stop then next + else if h : i + 1 < s.ranges.size then s.ranges[i + 1].start + else r.stop + +/-- +Concatenates the slices of `s.str` covered by each range intersected with `[p1, p2)`, in source +order. This is equivalent to walking via `get`/`next` from `p1` to `p2` (skipping inter-range gaps), +but it can be more efficient. +-/ +def extract (s : InlineSource) (p1 p2 : String.Pos.Raw) : String := + s.ranges.foldl (init := "") fun acc r => + let lo : String.Pos.Raw := ⟨max p1.byteIdx r.start.byteIdx⟩ + let hi : String.Pos.Raw := ⟨min r.stop.byteIdx p2.byteIdx⟩ + if lo < hi then acc ++ String.Pos.Raw.extract s.str lo hi + else acc + +/-- +Constructs an `InlineSource` from a source string and the line ranges of a paragraph block. This is +typically the output of `splitLines` plus container- prefix consumption. + +It implements the CommonMark §4.8 paragraph-content normalisation directly on file positions: + +- Leading spaces and tabs are stripped from each line's start. +- Each non-final line's stop position is extended by one byte when the source has a `'\n'` there. +- Trailing spaces/tabs/`\n`/`\r` are stripped from the *final* line's end, so trailing whitespace + cannot leak into the rendered output. +-/ +def ofLines (str : String) (lines : Array Syntax.Range) : InlineSource := Id.run do + let mut ranges : Array Syntax.Range := Array.emptyWithCapacity lines.size + for h : idx in [0 : lines.size] do + let line := lines[idx] + let lineSub : Substring.Raw := { str, startPos := line.start, stopPos := line.stop } + let start := lineSub.trimLeft.startPos + let stop : String.Pos.Raw := + let isNonLastNewline := + idx + 1 < lines.size && + line.stop < str.rawEndPos && + line.stop.get str == '\n' + if isNonLastNewline then + line.stop + '\n' + else line.stop + ranges := ranges.push { start, stop } + -- Trim trailing whitespace from the last range (CommonMark §4.8 final-WS removal). + if h : ranges.size > 0 then + let lastIdx := ranges.size - 1 + have : ranges.size - 1 < ranges.size := Nat.sub_one_lt_of_lt h + let r := ranges[lastIdx] + let lastSub : Substring.Raw := { str, startPos := r.start, stopPos := r.stop } + ranges := ranges.set lastIdx { r with stop := lastSub.trimRight.stopPos } + return { str, ranges } + +/-- +A single-range `InlineSource` over `range` of `str`. Used for content that is naturally a single +contiguous source span (e.g. an ATX heading's content slice). +-/ +def ofRange (str : String) (range : Syntax.Range) : InlineSource := + { str, ranges := #[range] } + +/-- +Returns an `InlineSource` whose ranges represent the suffix of `s` that starts at position `p`. +Ranges fully before `p` are dropped and the range containing `p` (if any) is trimmed to start at +`p`. +-/ +def dropUpTo (s : InlineSource) (p : String.Pos.Raw) : InlineSource := Id.run do + let mut out : Array Syntax.Range := #[] + for r in s.ranges do + if r.stop ≤ p then continue + if r.start < p then + out := out.push { start := p, stop := r.stop } + else + out := out.push r + return { str := s.str, ranges := out } + +end InlineSource + +/-- +A bounded sub-view of an `InlineSource`, analogous to `Substring.Raw` +over a `String`: it carries the parent source plus an explicit +`startPos` / `stopPos` range. +-/ +structure InlineRange where + /-- The parent source. -/ + source : InlineSource + /-- The current logical position within `source`. -/ + startPos : String.Pos.Raw + /-- One past the last logical position covered by this view. -/ + stopPos : String.Pos.Raw +deriving Inhabited + +/-- +Builds an `InlineRange` viewing `[startPos, stopPos)` of `s`. +-/ +@[inline] def InlineSource.range (s : InlineSource) (startPos stopPos : String.Pos.Raw) : + InlineRange := + { source := s, startPos, stopPos } + +namespace InlineRange + +/-- Returns `r` with its `startPos` replaced by `p` (and the same source/stop). -/ +@[inline] def withStart (r : InlineRange) (p : String.Pos.Raw) : InlineRange := + { r with startPos := p } + +/-- Whether this range has been fully consumed. -/ +@[inline] def isEmpty (r : InlineRange) : Bool := + r.startPos ≥ r.stopPos + +/-- The character at the range's `startPos`. -/ +@[inline] def front (r : InlineRange) : Char := + r.source.get r.startPos + +/-- +Advances `startPos` by one logical character. +-/ +@[inline] def drop1 (r : InlineRange) : InlineRange := + { r with startPos := r.source.next r.startPos } + +/-- +If `r` begins with `c`, returns the range advanced past it. Otherwise returns `none`. +-/ +@[inline] def expectChar (c : Char) (r : InlineRange) : Option InlineRange := + if !r.isEmpty && r.front == c then some r.drop1 else none + +/-- +If `r` begins with `c`, returns the source range that covers the character together with `r` +advanced past `c`. Otherwise returns `none`. +-/ +@[inline] def matchCharRange (c : Char) (r : InlineRange) : Option (Syntax.Range × InlineRange) := + if !r.isEmpty && r.front == c then + let r' := r.drop1 + some ({ start := r.startPos, stop := r'.startPos }, r') + else none + +/-- +Advances past a run of `c`s at the start of `r`. Returns the post-run range and the run length. +-/ +partial def matchRun (c : Char) (r : InlineRange) : InlineRange × Nat := + go r 0 +where + go (r : InlineRange) (n : Nat) : InlineRange × Nat := + if !r.isEmpty && r.front == c then go r.drop1 (n + 1) else (r, n) diff --git a/src/Lean/Server/FileWorker/Markdown/Parser.lean b/src/Lean/Server/FileWorker/Markdown/Parser.lean new file mode 100644 index 000000000000..1abab3e1bf5e --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser.lean @@ -0,0 +1,57 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Lean.Server.FileWorker.Markdown.Parser.Block +public import Lean.Server.FileWorker.Markdown.Parser.Inline + +namespace Lean.Server.FileWorker.Markdown + + +/-- +Parses the inline content of the blocks identified in a Markdown document using `parseInlines`. This +is the second pass of the algorithm suggested in the CommonMark specification. Two passes are +necessary because the presence of a link definition later in the document affects the syntactic +interpretation of reference-style links earlier in the document. +-/ +partial def parseInlinesInBlock (refs : RefTable) (s : String) : + Block Syntax.Range → Block (Array Inline) + | .paragraph lines => + let src := InlineSource.ofLines s lines + .paragraph #[parseInlines refs src { start := src.startPos, stop := src.stopPos }] + | .atxHeading hashes content closeHashes? => + .atxHeading hashes (parseInlines refs (.ofRange s content) content) closeHashes? + | .setextHeading level lines underline => + let src := InlineSource.ofLines s lines + .setextHeading level + #[parseInlines refs src { start := src.startPos, stop := src.stopPos }] + underline + | .fencedCode info openFence closeFence? lines => + .fencedCode info openFence closeFence? lines + | .indentedCode lines => + .indentedCode lines + | .blockquote markers children => + .blockquote markers (children.map (parseInlinesInBlock refs s)) + | .list kind tight items => + .list kind tight (items.map (parseInlinesInBlock refs s)) + | .listItem marker children => + .listItem marker (children.map (parseInlinesInBlock refs s)) + | .linkRefDef m => + .linkRefDef m + | .thematicBreak line => + .thematicBreak line + +/-- +Parses a Markdown string, delimited by `startPos` and `endPos`. + +This parser is based on the two-pass algorithm specified in the CommonMark specification. +-/ +public def parseDocument (s : String) (startPos endPos : String.Pos.Raw) : + Array (Block (Array Inline)) := + let blocks := parseBlocks s startPos endPos + let refs := buildRefTable s blocks + blocks.map (parseInlinesInBlock refs s) diff --git a/src/Lean/Server/FileWorker/Markdown/Parser/Block.lean b/src/Lean/Server/FileWorker/Markdown/Parser/Block.lean new file mode 100644 index 000000000000..a30f9d0b824b --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser/Block.lean @@ -0,0 +1,1194 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Init.Data.Nat.Fold +public import Init.Data.String.TakeDrop +public import Lean.Server.FileWorker.Markdown.Basic +public import Lean.Server.FileWorker.Markdown.InlineSource +import Lean.Server.FileWorker.Markdown.Parser.Cursor +import Lean.Server.FileWorker.Markdown.Parser.BlockZipper + +namespace Lean.Server.FileWorker.Markdown + +/-! +# Pass 1: Block Parser + +CommonMark is parsed in two passes: +1. The first pass identifies the block structure of the document and accumulates the definitions of + ref-style links. +2. Within each block, the second pass finds the structure of inlines and applies fallback rules for + poorly nested or mismatched delimiters. + +This file contains the first pass. It is almost compliant with CommonMark; unlike CommonMark, it +does not support HTML blocks, and case folding of ref-style link definitions is only for ASCII +`a`-`z`. +-/ + +/-- +Counts and skips leading ASCII spaces in `sub`. Returns the count and the new byte position. +-/ +def countSpaces (sub : Substring.Raw) : Nat × String.Pos.Raw := Id.run do + let mut sub := sub + let mut n := 0 + while !sub.isEmpty && sub.front == ' ' do + n := n + 1 + sub := sub.drop 1 + return (n, sub.startPos) + +/-- +Counts tab-aware leading whitespace (CommonMark §2.2): each tab advances to the next 4-column tab +stop, computed relative to the cursor's current column. Returns the advanced cursor and the total +number of columns consumed. Tabs are always crossed in full. +-/ +def countIndentCols (c : Cursor) : Cursor × Nat := Id.run do + let mut sub := c.rest + let mut cols := 0 + while !sub.isEmpty do + let ch := sub.front + if ch == ' ' then + cols := cols + 1 + sub := sub.drop 1 + else if ch == '\t' then + let curCol := c.col + cols + cols := cols + (4 - curCol % 4) + sub := sub.drop 1 + else + break + return ({ rest := sub, col := c.col + cols }, cols) + +/-- +Builds a `String` of `n` ASCII spaces. +-/ +def spaces (n : Nat) : String := + n.fold (init := "") fun _ _ s => s.push ' ' + +/-- +Consumes up to `maxCols` columns of leading whitespace at `c`, with tab-aware column counting. +Returns the advanced cursor and the number of columns actually consumed. + +Tabs may be split: if a tab would carry the cursor past `maxCols`, the byte position stays at the +tab and the column count advances by exactly the requested amount, leaving “leftover” tab columns +for subsequent consumers (CommonMark §2.2's partial-tab rule). +-/ +def consumeIndentCapped (c : Cursor) (maxCols : Nat) : Cursor × Nat := Id.run do + let mut sub := c.rest + let mut cols := 0 + while !sub.isEmpty && cols < maxCols do + let ch := sub.front + if ch == ' ' then + cols := cols + 1 + sub := sub.drop 1 + else if ch == '\t' then + let curCol := c.col + cols + let advance := 4 - curCol % 4 + if cols + advance > maxCols then + -- Partial tab: byteIdx stays at the tab; cols advances by the + -- requested amount only. The remaining tab cols are observable to + -- the next consumer through the cursor's (byteIdx, col) pair. + cols := maxCols + else + cols := cols + advance + sub := sub.drop 1 + else + break + return ({ rest := sub, col := c.col + cols }, cols) + +/-- +Converts a line of an indented code block into the string that it denotes. + +Renders a line of indented-code content: strips up to 4 columns of leading indentation (relative to +the cursor's current column) and materializes any partial-tab leftover as ASCII spaces (CommonMark +§2.2). The byte cursor may land mid-tab when the indentation doesn't fall on a tab-stop boundary; in +that case, the leftover columns of that tab become spaces in the rendered output. +-/ +def indentedCodeLineToString (c : Cursor) : String := Id.run do + let (c', _) := consumeIndentCapped c 4 + let p := c'.byteIdx + let s := c.str + let mut leftoverPrefix := "" + let mut contentStart := p + if !c'.rest.isEmpty && c'.rest.front == '\t' then + if c'.col % 4 != 0 then + leftoverPrefix := spaces (4 - c'.col % 4) + contentStart := (c'.rest.drop 1).startPos + return leftoverPrefix ++ String.Pos.Raw.extract s contentStart c.stopPos + +/-- Whether `sub` is blank (only spaces or tabs). -/ +def isBlankSlice (sub : Substring.Raw) : Bool := + sub.all fun c => c == ' ' || c == '\t' + +/-- Drops trailing spaces and tabs from `sub` and returns the new end position. -/ +def trimTrailingWs (sub : Substring.Raw) : String.Pos.Raw := + (sub.dropRightWhile fun c => c == ' ' || c == '\t').stopPos + +/-- Splits `sub` into line ranges, each excluding the trailing `'\n'`. -/ +def splitLines (sub : Substring.Raw) : Array Syntax.Range := Id.run do + let mut lines : Array Syntax.Range := #[] + let mut sub := sub + let mut lineStart := sub.startPos + while !sub.isEmpty do + if sub.front == '\n' then + lines := lines.push { start := lineStart, stop := sub.startPos } + sub := sub.drop 1 + lineStart := sub.startPos + else + sub := sub.drop 1 + if lineStart < sub.startPos then + lines := lines.push { start := lineStart, stop := sub.startPos } + return lines + +/-- +A successful ATX heading match. +-/ +structure AtxHeadingMatch where + /-- The heading level (1-6). -/ + level : Nat + /-- The source range of the opening `#`s. -/ + hashes : Syntax.Range + /-- The trimmed inline content range. -/ + content : Syntax.Range + /-- The optional closing hashes. -/ + closeHashes : Option Syntax.Range + +/-- +Tries to recognize an ATX heading at `c`. Trailing whitespace is trimmed from `content`. +-/ +def matchAtxHeading (c : Cursor) : Option AtxHeadingMatch := do + let (c, indent) := countIndentCols c + guard (indent ≤ 3) + let s := c.str + let hashStart := c.byteIdx + -- CommonMark §4.2 caps ATX heading levels at 6. We scan up to 7 hashes so + -- runs of length ≥ 7 fall through the `count ≤ 6` guard below as non-headings. + let mut rest := c.rest + let mut count := 0 + while !rest.isEmpty && count < 7 do + if rest.front != '#' then break + count := count + 1 + rest := rest.drop 1 + guard (count > 0 && count ≤ 6) + if !rest.isEmpty then + let next := rest.front + guard (next == ' ' || next == '\t') + let q := rest.startPos + let hashes : Syntax.Range := { start := hashStart, stop := q } + -- Skip whitespace (spaces or tabs) between the hashes and the content. + let (cAfterGap, _) := countIndentCols (c.moveTo q 0) + let contentStart := cAfterGap.byteIdx + let mut contentEnd := trimTrailingWs { c.rest with startPos := contentStart } + -- Optional closing `#` sequence: a trailing run of `#` characters that + -- is either preceded by a space/tab or extends to the start of content + -- (CommonMark §4.2). Strip it from the content range. + let mut closeStart := contentEnd + while contentStart < closeStart + && (String.Pos.Raw.prev s closeStart).get s == '#' do + closeStart := String.Pos.Raw.prev s closeStart + let closeLen := contentEnd.byteIdx - closeStart.byteIdx + let mut closeHashes? : Option Syntax.Range := none + if closeLen > 0 then + if closeStart == contentStart then + closeHashes? := some { start := closeStart, stop := contentEnd } + contentEnd := closeStart + else + let beforeClose := String.Pos.Raw.prev s closeStart + let cb := beforeClose.get s + if cb == ' ' || cb == '\t' then + closeHashes? := some { start := closeStart, stop := contentEnd } + contentEnd := trimTrailingWs + { str := s, startPos := contentStart, stopPos := closeStart } + return { + level := count + hashes + content := { start := contentStart, stop := contentEnd } + closeHashes := closeHashes? + } + +/-- +Whether the line at `c` is a thematic break: up to 3 leading spaces, followed by 3+ matching +`*`/`-`/`_` characters, with only spaces or tabs allowed between or after them (CommonMark §4.1). +-/ +def matchThematicBreak (c : Cursor) : Bool := Id.run do + let (c, indent) := countIndentCols c + if indent > 3 then return false + let some (markerChar, _) := c.peek? | return false + if markerChar != '*' && markerChar != '-' && markerChar != '_' then + return false + let mut rest := c.rest + let mut markerCount := 0 + while !rest.isEmpty do + let ch := rest.front + if ch == markerChar then + markerCount := markerCount + 1 + rest := rest.drop 1 + else if ch == ' ' || ch == '\t' then + rest := rest.drop 1 + else + return false + return markerCount >= 3 + +/-- +Recognizes a setext heading underline (CommonMark §4.3): up to three leading spaces, then a run of +`=` (level 1) or `-` (level 2), then optional trailing whitespace. Returns the heading level on +success. + +Note: a `---` underline overlaps with the thematic-break syntax. The caller must enforce the +precedence rule: setext header parsing applies only when an open paragraph immediately precedes the +underline; otherwise, the same line is interpreted as a thematic break. +-/ +def matchSetextUnderline (c : Cursor) : Option Nat := do + let (c, indent) := countIndentCols c + guard (indent ≤ 3) + let (markerChar, _) ← c.peek? + guard (markerChar == '=' || markerChar == '-') + let (cAfterRun, _) := c.advanceWhile (· == markerChar) + -- After the run, only whitespace is permitted to end of line. + let (cAfterWs, _) := cAfterRun.advanceWhile fun ch => + ch == ' ' || ch == '\t' + guard (cAfterWs.byteIdx ≥ cAfterWs.stopPos) + return if markerChar == '=' then 1 else 2 + +/-- Tries to recognize an opening code fence at `c`. -/ +def matchFenceOpen (c : Cursor) : Option (FenceInfo × Syntax.Range) := do + let (c, indent) := countIndentCols c + guard (indent ≤ 3) + let s := c.str + let (fenceChar, _) ← c.peek? + let fenceChar ← FenceChar.ofChar? fenceChar + let fenceStart := c.byteIdx + let (cAfterFence, count) := c.advanceWhile (· == fenceChar.toChar) + guard (count ≥ 3) + let q := cAfterFence.byteIdx + let openFence : Syntax.Range := { start := fenceStart, stop := q } + let (_, infoStart) := countSpaces { c.rest with startPos := q } + let infoEnd := trimTrailingWs { c.rest with startPos := infoStart } + -- A backtick fence's info string may not contain backticks. + if fenceChar == .backtick then + let mut eRest : Substring.Raw := { str := s, startPos := infoStart, stopPos := infoEnd } + while !eRest.isEmpty do + let ch := eRest.front + guard (ch != '`') + eRest := eRest.drop 1 + let infoString? : Option Syntax.Range := + if infoStart < infoEnd then some { start := infoStart, stop := infoEnd } else none + return ({ fenceChar, fenceLen := count, openIndent := indent, infoString? }, + openFence) + +/-- +Tries to recognize a closing fence on a line whose container prefixes have already been consumed. +The closing fence must match `info`'s character, be at least as long as the opener, and contain only +trailing whitespace afterwards. +-/ +def matchFenceClose (line : Substring.Raw) (info : FenceInfo) : Option Syntax.Range := do + let stop := line.stopPos + let (indent, p) := countSpaces line + guard (indent ≤ 3) + let pRest : Substring.Raw := { line with startPos := p } + guard <| !pRest.isEmpty && pRest.front == info.fenceChar.toChar + let cAtFence : Cursor := { rest := pRest, col := indent } + let (cAfterFence, count) := cAtFence.advanceWhile (· == info.fenceChar.toChar) + guard <| count ≥ info.fenceLen + let q := cAfterFence.byteIdx + let (_, after) := countSpaces { line with startPos := q } + guard <| after ≥ stop + return { start := p, stop := q } + +/-- +Tries to recognize a single blockquote marker (`>` plus optional space). Returns the marker's source +range and a cursor positioned just past the consumed prefix (with column adjusted, possibly mid-tab +per CommonMark §2.2's partial-tab rule). +-/ +def matchBlockquoteMarker (c : Cursor) : Option (Syntax.Range × Cursor) := do + let (c, indent) := countIndentCols c + guard (indent ≤ 3) + let p := c.byteIdx + guard (!c.rest.isEmpty && c.rest.front == '>') + let afterMarker := c.rest.drop 1 + let mEnd := afterMarker.startPos + -- The blockquote marker `>` is followed by an optional single space (or + -- one column of a tab — partial-tab consumption per CommonMark §2.2). + -- `countIndentCols` already advanced `c.col` past the leading indent. + let baseCol := c.col + 1 -- col just after the `>` + if !afterMarker.isEmpty then + let nextC := afterMarker.front + let mEndNext := (afterMarker.drop 1).startPos + if nextC == ' ' then + return ({ start := p, stop := mEnd }, c.moveTo mEndNext (baseCol + 1)) + if nextC == '\t' then + let tabStop := baseCol + (4 - baseCol % 4) + if tabStop == baseCol + 1 then + -- Tab equivalent to one column; cross it fully. + return ({ start := p, stop := mEnd }, c.moveTo mEndNext (baseCol + 1)) + else + -- Partial tab: leave the byte in place but advance the column. + return ({ start := p, stop := mEnd }, c.moveTo mEnd (baseCol + 1)) + return ({ start := p, stop := mEnd }, c.moveTo mEnd baseCol) + +/-- +A successful list marker match. +-/ +structure ListMarkerMatch where + /-- The source range of the marker itself. -/ + marker : Syntax.Range + /-- A cursor positioned just past the absorbed prefix. -/ + after : Cursor + /-- The marker's kind.-/ + kind : ListKind + /-- + The column distance from the input cursor to `after`, which is the minimum indentation of + subsequent continuation lines. + -/ + contentColumn : Nat + +/-- Tries to recognize a bullet or ordered list marker at `c`. -/ +def matchListMarker (c : Cursor) : Option ListMarkerMatch := do + let initialCol := c.col + let (c, indent) := countIndentCols c + guard (indent ≤ 3) + let stop := c.stopPos + let p := c.byteIdx + guard !c.rest.isEmpty + let ch := c.rest.front + let pNext := (c.rest.drop 1).startPos + let preMarkerCol := c.col + -- After matching the marker, compute the listItem's content column. Per + -- CommonMark §5.2: 1 ≤ N ≤ 4 cols of whitespace following the marker are + -- absorbed into the listItem prefix; if the line is empty after the marker + -- or has 5+ cols of whitespace (i.e., the content is itself indented + -- code), the listItem absorbs only one col of whitespace. Tabs may be + -- partially consumed (CommonMark §2.2). + let determineContent (markerEnd : String.Pos.Raw) (markerEndCol : Nat) : Option Cursor := do + let baseCursor := c.moveTo markerEnd markerEndCol + if markerEnd >= stop then + return c.moveTo markerEnd (markerEndCol + 1) + guard !baseCursor.rest.isEmpty + let nextC := baseCursor.rest.front + guard <| nextC == ' ' || nextC == '\t' + let (afterWsCursor, wsCols) := countIndentCols baseCursor + let contentBlank := afterWsCursor.byteIdx >= stop + let absorbCols : Nat := + if contentBlank || wsCols >= 5 then 1 + else wsCols + let (cursorAfter, _) := consumeIndentCapped baseCursor absorbCols + return cursorAfter + -- Bullet markers + if ch == '*' || ch == '-' || ch == '+' then + let markerEnd := pNext + let markerEndCol := preMarkerCol + 1 + let kind : BulletKind := + if ch == '*' then .star + else if ch == '-' then .hyphen + else .plus + let cursorAfter ← determineContent markerEnd markerEndCol + return { + marker := { start := p, stop := markerEnd } + after := cursorAfter + kind := .bullet kind + contentColumn := cursorAfter.col - initialCol + } + -- Ordered markers + if ch.isDigit then + let mut rest := c.rest + let mut digits := 0 + let mut value : Nat := 0 + -- CommonMark §5.2: an ordered-list marker has 1–9 digits. We scan up to + -- 10 so runs of length ≥ 10 fall through the `digits ≤ 9` guard below. + while !rest.isEmpty && digits < 10 do + let digit := rest.front + if !digit.isDigit then break + value := value * 10 + (digit.toNat - '0'.toNat) + digits := digits + 1 + rest := rest.drop 1 + guard (digits != 0) + guard (digits ≤ 9) + guard !rest.isEmpty + let delim := rest.front + guard (delim == '.' || delim == ')') + let markerEnd := (rest.drop 1).startPos + let markerEndCol := preMarkerCol + digits + 1 + let orderedDelim : OrderedDelim := if delim == '.' then .dot else .rparen + let cursorAfter ← determineContent markerEnd markerEndCol + return { + marker := { start := p, stop := markerEnd } + after := cursorAfter + kind := .ordered value orderedDelim + contentColumn := cursorAfter.col - initialCol + } + failure + +/-- +Whether a list marker matched at the current cursor is allowed to interrupt an open paragraph +(CommonMark §5.2): + * An empty list item cannot interrupt a paragraph + * An ordered list marker can interrupt only if its start number is `1`. + +`after` is positioned just past the marker. +-/ +def listMarkerInterruptsParagraph (after : Cursor) (kind : ListKind) : Bool := + !isBlankSlice after.rest + && match kind with + | .ordered start _ => start == 1 + | .bullet _ => true + +/-- +Whether `kind` is the same kind of list as one already open in a parent. A marker of the same kind +is always allowed to start a new sibling item, even when it would otherwise violate +`listMarkerInterruptsParagraph` (CommonMark §5.2's strict rules apply only when *opening* a list, +not when extending one). +-/ +def hasSameKindListAncestor (z : BlockZipper) (kind : ListKind) : Bool := + z.spine.any fun frame => + match frame.container with + | .list k => k.sameKind kind + | _ => false + +/-- +Whether the deepest open list has a *different* kind than `kind`. A marker of a different kind +closes that list (and the listItem and any leaf inside it), so it effectively “interrupts” any +paragraph that was open. The strict §5.2 start-≠-1 rule does not block this, since after the list +closes the new list opens at the parent level, where there is no longer a paragraph to interrupt. +-/ +def closesEnclosingList (z : BlockZipper) (kind : ListKind) : Bool := + z.spine.findSomeRev? differingListKind |>.getD false +where + differingListKind (frame : ZipperFrame) : Option Bool := + match frame.container with + | .list k => some (!k.sameKind kind) + | _ => none + +/-- +Whether the line at `cursor` would *interrupt* a paragraph. In other words, it would open some other +kind of block such that lazy continuation rules cannot apply. + +Indented code blocks are intentionally not interrupts (CommonMark §4.4: an indented code block +cannot interrupt a paragraph). List markers must additionally satisfy +`listMarkerInterruptsParagraph` unless they are a sibling of an already-open list with the same +marker kind. +-/ +def isInterrupt (z : BlockZipper) (c : Cursor) : Bool := + isBlankSlice c.rest || + (matchBlockquoteMarker c).isSome || + (match matchListMarker c with + | some m => + hasSameKindListAncestor z m.kind || + closesEnclosingList z m.kind || + listMarkerInterruptsParagraph m.after m.kind + | none => false) || + (matchAtxHeading c).isSome || + (matchFenceOpen c).isSome || + matchThematicBreak c + +/-! +# Processing Lines + +Lines are processed one at a time. A kind of rightward-only zipper is maintained, allowing blocks to +be added at the lower right of the syntax tree. + +Each step of the pipeline is a `ProcessM Unit` named after its description in the CommonMark spec. +Steps that produce the line's final zipper short-circuit the pipeline by `throw`-ing it, while steps +that only mutate the in-flight state fall through to the next. +-/ + +/-- +The state that flows through the late (post-close) steps of `processLine`. +-/ +private structure LineState where + /-- The in-progress document. -/ + z : BlockZipper + /-- The cursor that advances through the block-inducing prefixes of the line. -/ + c : Cursor + /-- Whether the preceding line was blank.-/ + prevBlank : Bool + +/-- +The state during the early container-continuation phase that tracks the length of the suffix of the +zipper's spine (that is, parent blocks) that are continued on this line. +-/ +private structure ContState extends LineState where + /-- The number of inner spine frames that continued on this line. -/ + matched : Nat + /-- The matched counter is always a valid prefix length of the spine. -/ + matched_le : matched ≤ z.spine.size + +/-- +The monad for line processing after containers that need closing have been closed. + +The “exception” channel carries the line's final `BlockZipper`, and is used for early return. +-/ +private abbrev ProcessM := EStateM BlockZipper LineState + +/-- +The monad for determining which containers should continue for a given line. +-/ +private abbrev ContainerM := EStateM BlockZipper ContState + +/-- +Lifts a `ContainerM` action to `ProcessM`. +-/ +private def runContainerM (act : ContainerM Unit) : ProcessM Nat := do + let s ← get + let init : ContState := + { s with matched := 0, matched_le := Nat.zero_le _ } + match act.run init with + | .ok _ s' => set s'.toLineState; return s'.matched + | .error z _ => throw z + +/-- +Walks the spine, advancing the cursor past the markers of containers that continue. Updates the +zipper in place (e.g. pushing markers into blockquote frames) and writes `matched` — the number of +inner spine frames that continued, so subsequent steps can close the rest. The document root +continues implicitly and is not counted. +-/ +private def continueContainers : ContainerM Unit := do + let mut s ← get + let mut keepWalking := true + while h : keepWalking ∧ s.matched < s.z.spine.size do + let frame := s.z.spine[s.matched]'h.2 + match frame.container with + | .blockquote markers => + match matchBlockquoteMarker s.c with + | none => keepWalking := false + | some (markerRange, cAfter) => + let frame' := { frame with container := .blockquote (markers.push markerRange) } + s := { s with + c := cAfter + z := { s.z with spine := s.z.spine.set s.matched frame' h.2 } + matched := s.matched + 1 + matched_le := by cases h; simpa + } + | .list _ => + -- Lists themselves carry no continuation marker; their listItem children do. + s := { s with matched := s.matched + 1, matched_le := h.2 } + | .listItem _ contentColumn => + -- Try to consume `contentColumn` cols of indent (in cols, tab-aware + -- with partial-tab support). + let (cAfter, consumed) := consumeIndentCapped s.c contentColumn + if consumed ≥ contentColumn then + s := { s with c := cAfter, matched := s.matched + 1, matched_le := h.2 } + else if isBlankSlice s.c.rest then + -- A blank line continues the listItem (cursor not advanced). + s := { s with matched := s.matched + 1, matched_le := h.2 } + else + keepWalking := false + set s + +/-- +Tightness (CommonMark §5.3): if the previous line was blank and the deepest open `listItem` in the +matched range is being extended on this line, mark the *innermost* enclosing list as loose. Outer +list items see the blank as part of their nested child's body, not as a separator between their own +direct children. +-/ +private def markEnclosingListLoose : ContainerM Unit := do + let s ← get + unless s.prevBlank do return + -- Indices within the matched range live in `Fin s.matched`, and combined + -- with `s.matched_le` are valid spine indices. The walk goes from + -- `s.matched - 1` down to `0`, breaking on the first listItem found. + if h0 : 0 < s.matched then + let mut i : Fin s.matched := ⟨s.matched - 1, Nat.sub_one_lt_of_lt h0⟩ + let mut keepGoing := true + while keepGoing do + have h_i : i.val < s.z.spine.size := Nat.lt_of_lt_of_le i.isLt s.matched_le + if let .listItem _ _ := (s.z.spine[i]).container then + if h_pos : 0 < i.val then + have h_im1 : ↑i - 1 < s.z.spine.size := Nat.sub_lt_of_lt h_i + if let .list _ := (s.z.spine[↑i - 1]).container then + modify fun s' => + { s' with + z := + { s'.z with + spine := s'.z.spine.modify (i.val - 1) ({ · with loose := true }) } + matched_le := by cases s'; simpa + } + keepGoing := false + else if h_pos : 0 < i.val then + i := ⟨i.val - 1, Nat.sub_lt_of_lt i.isLt⟩ + else + keepGoing := false + +/-- +Checks whether to upgrade the preceding block to a Setext header. +-/ +private def trySetextHeading : ContainerM Unit := do + let s ← get + unless s.z.spine.size == s.matched do return + let some (.paragraph lines) := s.z.openLeaf? | return + let some level := matchSetextUnderline s.c | return + let underline : Syntax.Range := s.c.toEol + let z := { s.z with openLeaf? := none } + throw (z.addBlock (.setextHeading level lines underline)) + +/-- +Lazy paragraph continuation: if some containers failed to continue but we have an open paragraph and +the line is not an interrupt (i.e. doesn't open some other kind of block), the line lazily continues +the paragraph in its deepest open context. +-/ +private def tryLazyContinuation : ContainerM Unit := do + let s ← get + unless s.z.spine.size > s.matched do return + let some (.paragraph _) := s.z.openLeaf? | return + if isInterrupt s.z s.c then return + throw (s.z.extendParagraph s.c.toEol) + +/-- +Closes any spine containers that did not continue on this line. Takes the `matched` count produced +by the container continuation phase as an argument and closes the suffix of the parents that are +indicated. +-/ +private def closeNonContinuingContainers (matched : Nat) : ProcessM Unit := do + let s ← get + let mut z := s.z + while z.spine.size > matched do + z := z.closeContainer + set { s with z } + +/-- +Fenced code consumes lines verbatim until a matching closing fence appears. Either way, this step +finalizes the line. +-/ +private def consumeFencedCode : ProcessM Unit := do + let s ← get + let some (.fencedCode info _ _) := s.z.openLeaf? | return + match matchFenceClose s.c.rest info with + | some closeFence => throw (s.z.closeFencedCode closeFence) + | none => throw (s.z.extendFencedCode s.c.toEol) + +/-- +Indented code consumes 4+-indented lines verbatim and holds blank lines provisionally. Any other +line closes the block and falls through to the remaining steps. +-/ +private def consumeIndentedCode : ProcessM Unit := do + let s ← get + let some (.indentedCode _ _) := s.z.openLeaf? | return + if isBlankSlice s.c.rest then + let (cAfter, _) := consumeIndentCapped s.c 4 + let range : Syntax.Range := { start := cAfter.byteIdx, stop := s.c.stopPos } + throw (s.z.indentedCodeAddBlank range (indentedCodeLineToString s.c)) + let (_, indent) := countIndentCols s.c + if indent ≥ 4 then + let (cAfter, _) := consumeIndentCapped s.c 4 + let range : Syntax.Range := { start := cAfter.byteIdx, stop := s.c.stopPos } + throw (s.z.indentedCodeAddLine range (indentedCodeLineToString s.c)) + modify fun s => { s with z := s.z.closeLeaf } + +/-- +If the deepest container is a list whose previous item just closed and this line is non-blank +without a sibling marker, close the list so subsequent content doesn't become a stray child. +-/ +private def closeFinishedSiblingList : ProcessM Unit := do + let s ← get + let some (.list _) := s.z.top? | return + if !isBlankSlice s.c.rest && (matchListMarker s.c).isNone then + modify fun s => { s with z := s.z.closeContainer } + +/-- +Opens as many container prefixes (blockquote, list-item) as the line provides, in order. Returns +whether a list item was opened on this line, because the blank line handler downstream needs that +information to distinguish an empty list item's body from a blank-line separator. + +Thematic break has higher precedence than list markers (CommonMark §5.2: `* * *` is a thematic +break, not a one-item list), so each iteration short-circuits when a thematic break starts at the +current cursor. +-/ +private def openContainerPrefixes : ProcessM Bool := do + let mut openedListItemThisLine := false + let mut keepOpening := true + while keepOpening do + keepOpening := false + let s ← get + if matchThematicBreak s.c then break + if let some (markerRange, cAfter) := matchBlockquoteMarker s.c then + modify fun s => + { s with z := s.z.openContainer (.blockquote #[markerRange]), c := cAfter } + keepOpening := true + else if let some m := matchListMarker s.c then + -- The strict §5.2 rules (non-empty content, start-1 for ordered) apply + -- only when *opening* a new list. A same-kind marker that extends an + -- already-open list as a new sibling item is always allowed. + let interruptingPara := match s.z.openLeaf? with + | some (.paragraph _) => true + | _ => false + if interruptingPara && !hasSameKindListAncestor s.z m.kind + && !listMarkerInterruptsParagraph m.after m.kind then + keepOpening := false + else + let z' := + match s.z.top? with + | some (.list k) => + if k.sameKind m.kind then + -- Blank-line-then-new-sibling-item makes the list loose. + let z := + if s.prevBlank then + let lastIdx := s.z.spine.size - 1 + { s.z with + spine := s.z.spine.modify lastIdx fun f => { f with loose := true } } + else s.z + z.openContainer (.listItem m.marker m.contentColumn) + else + s.z.closeContainer + |>.openContainer (.list m.kind) + |>.openContainer (.listItem m.marker m.contentColumn) + | _ => + s.z.openContainer (.list m.kind) + |>.openContainer (.listItem m.marker m.contentColumn) + modify fun s => { s with z := z', c := m.after } + openedListItemThisLine := true + keepOpening := true + return openedListItemThisLine + +/-- +Parses a thematic break after any container prefixes have been consumed. +-/ +private def tryThematicBreak : ProcessM Unit := do + let s ← get + unless matchThematicBreak s.c do return + let mut z := s.z + while (match z.top? with | some (.list _) => true | _ => false) do + z := z.closeContainer + throw (z.addBlock (.thematicBreak s.c.toEol)) + +/-- +A blank line closes any open paragraph; list/blockquote frames stay open and may close at the next +non-continuing line. +-/ +private def tryBlankLine (openedListItemThisLine : Bool) : ProcessM Unit := do + let s ← get + unless isBlankSlice s.c.rest do return + let mut z := s.z.closeLeaf + if !openedListItemThisLine then + if let some lastFrame := z.spine.back? then + if let .listItem _ _ := lastFrame.container then + if lastFrame.closedChildren.isEmpty then + z := z.closeContainer + -- Only propagate “previous line was blank” when the deepest open container is itself a list item + -- (case A: blank inside an item's body) or a list (case B: my empty-item close just popped the + -- list item). Blanks deeper than a list item (e.g. inside a blockquote nested inside a listItem) + -- are *not* separators between siblings of the enclosing list and so don't make it loose. + let topIsListLike := + match z.spine.back?.map (·.container) with + | some (.listItem _ _) | some (.list _) => true + | _ => false + throw { z with lastWasBlank := topIsListLike && !openedListItemThisLine } + +/-- Parses an ATX heading -/ +private def tryAtxHeading : ProcessM Unit := do + let s ← get + let some m := matchAtxHeading s.c | return + throw (s.z.addBlock (.atxHeading m.hashes m.content m.closeHashes)) + +/-- Parses an opening code fence. -/ +private def tryFencedCodeOpener : ProcessM Unit := do + let s ← get + let some (info, openFence) := matchFenceOpen s.c | return + throw (s.z.openLeaf (.fencedCode info openFence #[])) + +/-- +Parses the start of an indented code block. +-/ +private def tryIndentedCodeOpener : ProcessM Unit := do + let s ← get + unless s.z.openLeaf?.isNone do return + let (_, indent) := countIndentCols s.c + unless indent ≥ 4 do return + throw (s.z.openLeaf (.indentedCode #[(s.c.toEol, indentedCodeLineToString s.c)] #[])) + +/-- +Starts a new paragraph, or extends the one that's open. + +This is the default parser if no others apply. +-/ +private def emitParagraph : ProcessM Unit := do + let s ← get + let lineFromCursor : Syntax.Range := s.c.toEol + match s.z.openLeaf? with + | some (.paragraph _) => throw (s.z.extendParagraph lineFromCursor) + | _ => throw (s.z.openLeaf (.paragraph #[lineFromCursor])) + +/-- +Processes a single source line, updating the zipper. +-/ +def processLine (z : BlockZipper) (line : Substring.Raw) : BlockZipper := + -- Capture and clear `lastWasBlank` at line entry so it represents "the + -- previous line was blank" for the duration of this call. + let prevBlank := z.lastWasBlank + let z : BlockZipper := { z with lastWasBlank := false } + let init : LineState := { z, c := Cursor.ofLine line, prevBlank } + let pipeline : ProcessM Unit := do + let matched ← runContainerM do + continueContainers + markEnclosingListLoose + trySetextHeading + tryLazyContinuation + closeNonContinuingContainers matched + consumeFencedCode + consumeIndentedCode + closeFinishedSiblingList + let openedListItemThisLine ← openContainerPrefixes + tryThematicBreak + tryBlankLine openedListItemThisLine + tryAtxHeading + tryFencedCodeOpener + tryIndentedCodeOpener + emitParagraph + match pipeline.run init with + | .ok _ s => s.z + | .error z _ => z + +/-- +Parses the block structure of a string, within the provided range. +-/ +def parseBlocksRaw (s : String) (startPos endPos : String.Pos.Raw) : + Array (Block Syntax.Range) := Id.run do + let mut z := BlockZipper.empty + for line in splitLines { str := s, startPos, stopPos := endPos } do + z := processLine z (rangeToSubstring line s) + return z.finalize +where + rangeToSubstring (r : Syntax.Range) (s : String) : Substring.Raw := + { str := s, startPos := r.start, stopPos := r.stop } + +/-- +Scans a link-reference-definition label `[label]` content (between `[` and `]`). Newlines are +permitted inside the label so long as no blank line appears. The label must contain at least one +non-whitespace character. CommonMark §4.7 caps labels at 999 *characters* before the closing `]`; we +approximate this by capping at 1000 source bytes (counting backslash escapes as 2). Returns the +label range and the cursor positioned at the closing `]`. +-/ +def scanRefDefLabel (r : InlineRange) : Option (Syntax.Range × InlineRange) := do + let labelStart := r.startPos + let mut r := r + let mut len := 0 + let mut sawNonWs := false + let mut sawNewline := false + while len < 1000 do + if r.isEmpty then break + let c := r.front + let rNext := r.drop1 + if c == '\\' && !rNext.isEmpty then + let esc := rNext.front + if esc != ' ' && esc != '\t' && esc != '\n' then sawNonWs := true + sawNewline := false + r := rNext.drop1 + len := len + 2 + else if c == ']' then + guard sawNonWs + return ({ start := labelStart, stop := r.startPos }, r) + else if c == '[' then + failure + else if c == '\n' then + guard !sawNewline -- blank line in label + sawNewline := true + r := rNext + len := len + 1 + else + if c != ' ' && c != '\t' then + sawNonWs := true + sawNewline := false + r := rNext + len := len + 1 + none + +/-- +Scans the destination of a link reference definition at the head of `r`. Recognizes the bracketed +`` and unbracketed forms. Returns the URL range (including any `<>` wrappers) and the cursor +positioned just past the destination. Fails on an unclosed `<`, an empty destination, unbalanced +parens, or an embedded `\n`. +-/ +def scanRefDefUrl (r : InlineRange) : Option (Syntax.Range × InlineRange) := do + guard !r.isEmpty + let urlStart := r.startPos + if r.front == '<' then + -- Bracketed: scan until the next `>`; reject `<` and `\n`. + let mut r := r.drop1 + let mut foundClose := false + while !r.isEmpty && !foundClose do + let c := r.front + let rNext := r.drop1 + if c == '\\' && !rNext.isEmpty then + r := rNext.drop1 + else if c == '>' then + r := rNext + foundClose := true + else if c == '<' || c == '\n' then + failure + else + r := rNext + guard foundClose + return ({ start := urlStart, stop := r.startPos }, r) + else + -- Unbracketed: stop at whitespace/control; balance parens. + let mut r := r + let mut parenDepth : Nat := 0 + let mut keepGoing := true + while keepGoing && !r.isEmpty do + let c := r.front + let rNext := r.drop1 + if c == '\\' && !rNext.isEmpty then + r := rNext.drop1 + else if c == ' ' || c == '\t' || c == '\n' then + keepGoing := false + else if c == '(' then + parenDepth := parenDepth + 1 + r := rNext + else if c == ')' then + if parenDepth == 0 then keepGoing := false + else + parenDepth := parenDepth - 1 + r := rNext + else if c.toNat < 0x20 then + keepGoing := false + else + r := rNext + guard <| r.startPos != urlStart -- non-empty destination + guard <| parenDepth == 0 + return ({ start := urlStart, stop := r.startPos }, r) + +/-- +Skips the whitespace gap between a link reference definition's `:` and its destination: spaces, +tabs, and at most one newline. Returns `none` if a blank line (two newlines) is encountered. +-/ +def skipRefDefDestGap (r : InlineRange) : Option InlineRange := do + let mut r := r + let mut newlinesSeen := 0 + while !r.isEmpty do + let c := r.front + if c == ' ' || c == '\t' then r := r.drop1 + else if c == '\n' then + newlinesSeen := newlinesSeen + 1 + if newlinesSeen ≥ 2 then failure + r := r.drop1 + else break + return r + +/-- +Consumes any same-line trailing whitespace from `r`, then an optional single `\n`. Returns the +cursor past the `\n` (or at end-of-buffer if the line was the last). Returns `none` if +non-whitespace content remains on the line. +-/ +def consumeRefDefLineEnd (r : InlineRange) : Option InlineRange := do + let mut r := r + while !r.isEmpty && (r.front == ' ' || r.front == '\t') do + r := r.drop1 + if r.isEmpty then return r + if r.front == '\n' then return r.drop1 + none + +/-- +Tries to consume an optional title following the destination of a link reference definition, plus +any trailing whitespace and the line's `\n`. Returns the title content range and the cursor past the +trailing `\n` (or end-of-buffer). + +The title may sit on the *same line* as the URL (separated by at least one space or tab and +delimited by `"…"`/`'…'`/`(…)`) or on the *next line*, after exactly one newline plus optional +indentation. + +A blank line inside the title is forbidden, as is an unescaped `(` inside a `(…)` title. After the +closing delimiter, only same-line whitespace and an optional trailing newline are permitted; +otherwise the title attempt is abandoned (`none`) so the caller can fall back to a no-title +interpretation. +-/ +def refDefTitle (r : InlineRange) : Option (Syntax.Range × InlineRange) := do + -- Locate the title opener: either same-line after whitespace, or on the + -- next line after a single newline plus optional leading whitespace. + let mut sawWs := false + let mut p := r + while !p.isEmpty && (p.front == ' ' || p.front == '\t') do + sawWs := true + p := p.drop1 + guard !p.isEmpty + let mut openerR : InlineRange := p + if p.front == '\n' then + openerR := p.drop1 + while !openerR.isEmpty && (openerR.front == ' ' || openerR.front == '\t') do + openerR := openerR.drop1 + else + guard sawWs + guard !openerR.isEmpty + let opener := openerR.front + guard <| opener == '"' || opener == '\'' || opener == '(' + -- Parse content up to a matching closer; reject blank lines and unescaped + -- `(` in `(...)` titles. + let closer : Char := if opener == '(' then ')' else opener + let afterOpener := openerR.drop1 + let titleContentStart := afterOpener.startPos + let mut tt := afterOpener + let mut titleContentEnd := titleContentStart + let mut foundCloser := false + let mut bail := false + let mut lastWasNewline := false + while !tt.isEmpty && !foundCloser && !bail do + let c := tt.front + let curPos := tt.startPos + let ttNext := tt.drop1 + if c == '\\' && !ttNext.isEmpty then + tt := ttNext.drop1 + lastWasNewline := false + else if c == closer then + titleContentEnd := curPos + foundCloser := true + tt := ttNext + else if c == opener && opener == '(' then + bail := true -- unescaped `(` inside a `(...)` title is forbidden + else if c == '\n' then + if lastWasNewline then bail := true + else + lastWasNewline := true + tt := ttNext + else + if c != ' ' && c != '\t' then lastWasNewline := false + tt := ttNext + guard (!bail && foundCloser) + let afterEol ← consumeRefDefLineEnd tt + return ({ start := titleContentStart, stop := titleContentEnd }, afterEol) + +/-- +Tries to parse a CommonMark link reference definition starting at `start` in `source`. Recognizes +both bracketed `` and unbracketed link destinations, and optional `"…"`/`'…'`/`(…)` titles +(which may appear on the line following the URL). The label may span multiple lines, but no blank +line is permitted inside it. + +Returns a `RefDefMatch` plus the position one past the last consumed character. +-/ +public def linkRefDef (source : InlineSource) (start : String.Pos.Raw) : + Option (RefDef × String.Pos.Raw) := do + -- 0–3 leading spaces of indentation. Tabs are *not* allowed at this + -- point — the source's per-line ranges have already been left-trimmed, + -- so any tab at the start of a line is part of the content. + let r0 : InlineRange := { source, startPos := start, stopPos := source.stopPos } + let (r, indent) := InlineRange.matchRun ' ' r0 + guard (indent ≤ 3) + -- `[label]`. + let (openBracket, r) ← InlineRange.matchCharRange '[' r + let (label, r) ← scanRefDefLabel r + let (closeBracket, r) ← InlineRange.matchCharRange ']' r + -- `:`. + let (colon, r) ← InlineRange.matchCharRange ':' r + -- Whitespace (including up to one newline) before the destination. + let r ← skipRefDefDestGap r + -- Destination. + let (url, afterUrl) ← scanRefDefUrl r + -- Optional title; on a syntactic failure or on trailing junk, fall back to + -- a no-title interpretation, in which case the URL itself must end the + -- line. + let (title?, finalRest) ← + (refDefTitle afterUrl |>.map fun (t, r) => (some t, r)) <|> + (consumeRefDefLineEnd afterUrl |>.map fun r => (none, r)) + return ({ openBracket, label, closeBracket, colon, url, title? }, finalRest.startPos) + +/-- +Greedily consumes link ref definitions from the front of `source`. + +Returns the extracted `linkRefDef` blocks and a residual `InlineSource`. +-/ +public def extractRefDefs (source : InlineSource) : + Array (Block Syntax.Range) × InlineSource := Id.run do + let mut defs : Array (Block Syntax.Range) := #[] + let mut p : String.Pos.Raw := source.startPos + let mut keep := true + while keep && p < source.stopPos do + match linkRefDef source p with + | none => keep := false + | some (m, posAfter) => + defs := defs.push (.linkRefDef m) + p := posAfter + return (defs, source.dropUpTo p) + +/-- + +Walks blocks, extracting link reference definitions off the front of each paragraph and setext-style +heading. Emits a `linkRefDef` block in source order for each successful extraction. + +If a setext heading's *entire* content is consumed by extraction, the heading is canceled. The +underline line is reinterpreted as paragraph text and merged with the following paragraph (if any) +without a blank separator, since they were originally one paragraph in source. A setext heading +whose preceding “paragraph” is just link reference definitions is not a heading at all. +-/ +partial def postProcessRefDefs (s : String) (blocks : Array (Block Syntax.Range)) : + Array (Block Syntax.Range) := Id.run do + let mut out : Array (Block Syntax.Range) := #[] + let mut i := 0 + while h : i < blocks.size do + let b := blocks[i] + match b with + | .paragraph lines => + let (defs, residual) := extractRefDefs (InlineSource.ofLines s lines) + if defs.isEmpty then + -- No ref-defs at the front: leave the paragraph untouched so its + -- original `lines` survive for downstream consumers (e.g. the + -- semantic-token highlighter). + out := out.push (.paragraph lines) + else + for d in defs do out := out.push d + unless residual.ranges.isEmpty do + out := out.push (.paragraph residual.ranges) + | .setextHeading level lines underline => + let (defs, residual) := extractRefDefs (InlineSource.ofLines s lines) + if defs.isEmpty then + out := out.push (.setextHeading level lines underline) + else if !residual.ranges.isEmpty then + for d in defs do out := out.push d + out := out.push (.setextHeading level residual.ranges underline) + else + for d in defs do out := out.push d + -- Heading canceled: treat the underline as a paragraph line and, + -- if the next block is a paragraph (no blank separator existed in + -- source), merge them so e.g. `[foo]: /url\n===\n[foo]` becomes + -- a single paragraph `===\n[foo]` after the link ref def is taken. + let mut combinedLines : Array Syntax.Range := #[underline] + if h : i + 1 < blocks.size then + if let .paragraph lines2 := blocks[i + 1] then + combinedLines := combinedLines ++ lines2 + i := i + 1 + let (defs2, residual2) := extractRefDefs (InlineSource.ofLines s combinedLines) + for d in defs2 do out := out.push d + unless residual2.ranges.isEmpty do + out := out.push (.paragraph residual2.ranges) + | .blockquote markers children => + let newChildren := postProcessRefDefs s children + out := out.push (.blockquote markers newChildren) + | .list kind tight items => + let newItems := postProcessRefDefs s items + out := out.push (.list kind tight newItems) + | .listItem marker children => + let newChildren := postProcessRefDefs s children + out := out.push (.listItem marker newChildren) + | other => out := out.push other + i := i + 1 + return out + +/-- +Builds the document's reference table. Earlier definitions of the same label take precedence. +-/ +public partial def buildRefTable (s : String) (blocks : Array (Block Syntax.Range)) : + RefTable := + go blocks |>.run {} |>.2 +where + go (bs : Array (Block Syntax.Range)) : StateM RefTable Unit := + bs.forM fun + | .linkRefDef m => do + let labelStr := String.Pos.Raw.extract s m.label.start m.label.stop + let urlRaw := String.Pos.Raw.extract s m.url.start m.url.stop + let urlStr := + if urlRaw.startsWith "<" && urlRaw.endsWith ">" then + let stripped := urlRaw.rawEndPos + String.Pos.Raw.extract urlRaw ⟨1⟩ ⟨stripped.byteIdx - 1⟩ + else urlRaw + let titleStr? := m.title?.map fun r => String.Pos.Raw.extract s r.start r.stop + let normLabel := normalizeLabel labelStr + if (← get).contains normLabel then pure () + else modify (·.insert normLabel { url := urlStr, title? := titleStr? }) + | .blockquote _ children => go children + | .list _ _ items => go items + | .listItem _ children => go children + | _ => pure () + +/-- +Identifies the complete block structure of a Markdown document. +-/ +public def parseBlocks (s : String) (startPos endPos : String.Pos.Raw) : + Array (Block Syntax.Range) := + postProcessRefDefs s (parseBlocksRaw s startPos endPos) diff --git a/src/Lean/Server/FileWorker/Markdown/Parser/BlockZipper.lean b/src/Lean/Server/FileWorker/Markdown/Parser/BlockZipper.lean new file mode 100644 index 000000000000..125d31f3efb0 --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser/BlockZipper.lean @@ -0,0 +1,253 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +import Init +public import Lean.Server.FileWorker.Markdown.Basic + +public section + +namespace Lean.Server.FileWorker.Markdown + +/-! +## The In-Progress Zipper + +While identifying the block structure of a document, lines are checked one at a time and added in +order. A zipper-like structure makes it easy to see the surrounding block context and to modify it +at the right-hand edge. +-/ + +/-- +A container that is currently open and may still receive child blocks. +-/ +inductive OpenContainer where + /-- + A blockquote. `markers` accumulates the positions of `>` markers from + every continuation line that has been seen so far. + -/ + | blockquote (markers : Array Syntax.Range) + /-- A list. -/ + | list (kind : ListKind) + /-- + A list item. + + - `marker` is this item's bullet or ordered-marker range, emitted as a + delimiter token at item-open time. + - `contentColumn` is the column at which the item's content begins; + subsequent lines must be indented to at least this column to count as + continuations of this item (CommonMark §5.2). + -/ + | listItem (marker : Syntax.Range) (contentColumn : Nat) +deriving Repr, Inhabited + +/-- +A frame of the zipper's spine: an open container plus the children that have +already been closed inside it, in source order. +-/ +structure ZipperFrame where + /-- + The container in this frame. Subsequent frames in the stack represent the last child of this open + container. + -/ + container : OpenContainer + /-- + Children that have already been closed inside this frame's container. + -/ + closedChildren : Array (Block Syntax.Range) := #[] + /-- + For list frames: set when a blank line followed by more content in this list has been observed + during parsing. + -/ + loose : Bool := false +deriving Inhabited + +/-- A leaf block that is currently open and may still accept additional source lines. -/ +inductive OpenLeaf where + /-- + An open paragraph being extended line by line. Closing the paragraph (because of a blank line, a + new block opener, or the parent container closing) finalizes it as `Block.paragraph`. + -/ + | paragraph (lines : Array Syntax.Range) + /-- + An open fenced code block. `lines` accumulates content lines until either a matching closing fence + appears (yielding `Block.fencedCode` with `closeFence? := some _`) or the parent container closes + at EOF (yielding `closeFence? := none`). + -/ + | fencedCode (info : FenceInfo) (openFence : Syntax.Range) (lines : Array Syntax.Range) + /-- + An open indented code block. + + - `lines` are the content lines committed so far. Each element is a pair of the line's source + range (covering the whole source line, including the indent the block consumed) and the + rendered string. In the rendered string, the 4 cols of indent that defined the code block + (relative to the parent container's content column) have already been stripped, with any + partial tabs materialized per CommonMark §2.2. + - `pendingBlanks` holds blank lines that may belong to this block: they are promoted to `lines` + when a subsequent indented line continues the block, and discarded when an unindented non-blank + line closes the block. This matches CommonMark's rule that blank lines between indented chunks + are part of the code block, but blank lines following the final chunk are not. + -/ + | indentedCode (lines : Array (Syntax.Range × String)) + (pendingBlanks : Array (Syntax.Range × String)) +deriving Inhabited + +/-- +The block parsing state. Maintains the spine of currently-open containers +plus an optional open leaf at the bottom. + +An empty spine means that only the document is open. + +Invariants (preserved by the zipper operations defined elsewhere): +- If `openLeaf?` is `some _`, that leaf logically lives inside the deepest + spine frame, or directly under the document if the spine is empty; + extending the leaf adds to its content, and closing the leaf pushes a + `Block` onto that container's children. +- A `OpenContainer.list` frame's immediate spine successor (when present) is + always an `OpenContainer.listItem`. +-/ +structure BlockZipper where + /-- Closed children of the document root, in source order. -/ + documentChildren : Array (Block Syntax.Range) := #[] + /-- Open containers below the document root, deepest last. -/ + spine : Array ZipperFrame := #[] + /-- An open leaf node, if one is present. -/ + openLeaf? : Option OpenLeaf := none + /-- + Whether the previous source line was blank (after consuming the current open containers' + prefixes). Used to mark enclosing lists as loose when a blank line is followed by content that + continues the list (CommonMark §5.3). + -/ + lastWasBlank : Bool := false +deriving Inhabited + +namespace BlockZipper + +/-- A zipper with only the document root open, no children, and no open leaf. -/ +def empty : BlockZipper := {} + +/-- The deepest currently-open container, or `none` if only the document is open. -/ +def top? (z : BlockZipper) : Option OpenContainer := + z.spine.back?.map (·.container) + +/-- +Converts an open leaf into its closed `Block` form. Fenced code blocks are not terminated with a +fence. +-/ +def leafToBlock : OpenLeaf → Block Syntax.Range + | .paragraph lines => .paragraph lines + | .fencedCode info openFence lines => .fencedCode info openFence none lines + | .indentedCode lines _pendingBlanks => .indentedCode lines + +/-- +Pushes a finalized block onto the deepest spine frame's `closedChildren`, or directly onto +`documentChildren` if the spine is empty. +-/ +private def pushChild (z : BlockZipper) (b : Block Syntax.Range) : BlockZipper := + if z.spine.isEmpty then + { z with documentChildren := z.documentChildren.push b } + else + { z with + spine := + z.spine.modify (z.spine.size - 1) + (fun frame => { frame with closedChildren := frame.closedChildren.push b }) } + +/-- Closes any open leaf, attaching it to the deepest container. -/ +def closeLeaf (z : BlockZipper) : BlockZipper := + match z.openLeaf? with + | none => z + | some leaf => + let z := z.pushChild (leafToBlock leaf) + { z with openLeaf? := none } + +/-- Pushes a new container onto the spine, closing any open leaf first. -/ +def openContainer (z : BlockZipper) (c : OpenContainer) : BlockZipper := + let z := z.closeLeaf + { z with spine := z.spine.push { container := c } } + +/-- Replaces the open leaf at the bottom of the spine, closing any prior one. -/ +def openLeaf (z : BlockZipper) (l : OpenLeaf) : BlockZipper := + let z := z.closeLeaf + { z with openLeaf? := some l } + +/-- +Adds a finalized block as a child of the deepest open container, closing any open leaf first. Used +for blocks like ATX headings that don't go through `OpenLeaf`. +-/ +def addBlock (z : BlockZipper) (b : Block Syntax.Range) : BlockZipper := + z.closeLeaf.pushChild b + +/-- Appends a source line to the open paragraph, if any. No-op otherwise. -/ +def extendParagraph (z : BlockZipper) (line : Syntax.Range) : BlockZipper := + match z.openLeaf? with + | some (.paragraph lines) => + { z with openLeaf? := some (.paragraph (lines.push line)) } + | _ => z + +/-- Appends a verbatim line to the open fenced code block, if any. No-op otherwise. -/ +def extendFencedCode (z : BlockZipper) (line : Syntax.Range) : BlockZipper := + match z.openLeaf? with + | some (.fencedCode info openFence lines) => + { z with openLeaf? := some (.fencedCode info openFence (lines.push line)) } + | _ => z + +/-- Closes the open fenced code block with a matching closing fence. -/ +def closeFencedCode (z : BlockZipper) (closeFence : Syntax.Range) : BlockZipper := + match z.openLeaf? with + | some (.fencedCode info openFence lines) => + let z := z.pushChild (.fencedCode info openFence (some closeFence) lines) + { z with openLeaf? := none } + | _ => z + +/-- +Appends an indented content line to the open indented code block. Any pending blanks held since the +last content line are promoted to real content lines first. +-/ +def indentedCodeAddLine (z : BlockZipper) (range : Syntax.Range) (rendered : String) : + BlockZipper := + match z.openLeaf? with + | some (.indentedCode lines pendingBlanks) => + let newLines := lines ++ pendingBlanks ++ #[(range, rendered)] + { z with openLeaf? := some (.indentedCode newLines #[]) } + | _ => z + +/-- +Holds a blank line provisionally inside the open indented code block. It is incorporated into the +block if a later indented line continues the block; otherwise, it is discarded when the block +closes. +-/ +def indentedCodeAddBlank (z : BlockZipper) (range : Syntax.Range) (rendered : String) : + BlockZipper := + match z.openLeaf? with + | some (.indentedCode lines pendingBlanks) => + { z with openLeaf? := some (.indentedCode lines (pendingBlanks.push (range, rendered))) } + | _ => z + +/-- Converts a frame into a finalized block. -/ +def frameToBlock (frame : ZipperFrame) : Block Syntax.Range := + match frame.container with + | .blockquote markers => .blockquote markers frame.closedChildren + | .list kind => .list kind (!frame.loose) frame.closedChildren + | .listItem marker _ => .listItem marker frame.closedChildren + +/-- +Pops the deepest container, finalizes it as a `Block`, and attaches it as a child +of its parent. This is a no-op when only the document root is open. +-/ +def closeContainer (z : BlockZipper) : BlockZipper := + let z := z.closeLeaf + match z.spine.back? with + | none => z + | some frame => + let z := { z with spine := z.spine.pop } + z.pushChild (frameToBlock frame) + +/-- Closes every container down to the document root and returns its children. -/ +def finalize (z : BlockZipper) : Array (Block Syntax.Range) := Id.run do + let mut z := z.closeLeaf + while !z.spine.isEmpty do + z := z.closeContainer + return z.documentChildren diff --git a/src/Lean/Server/FileWorker/Markdown/Parser/Cursor.lean b/src/Lean/Server/FileWorker/Markdown/Parser/Cursor.lean new file mode 100644 index 000000000000..49453443b1a7 --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser/Cursor.lean @@ -0,0 +1,96 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Init.Prelude +public import Init.While +public import Lean.Syntax + +public section + +namespace Lean.Server.FileWorker.Markdown + +/-! +## Line cursors + +The block parser walks one line at a time, and within each line it is constantly asking “what's at +column N relative to the parent container's indent?”. Because CommonMark counts indentation in +*columns* but the underlying source is bytes, and a single tab can advance the column by 1–4 +(CommonMark §2.2's partial-tab rule), byte position and column are not in lockstep. The cursor is a +structure that tracks both. +-/ + +/-- +A position within a single source line: an unconsumed slice plus the visual column its first byte +sits at. + +The pair `(rest.startPos, col)` is the parser's working position. They are kept in sync by the +column-aware advancement primitives below. Direct field updates should preserve the invariant that +`col` is the visual column of `rest.startPos`. +-/ +structure Cursor where + /-- The unconsumed suffix of the current line. -/ + rest : Substring.Raw + /-- + The visual column of `rest.startPos`. Computed under CommonMark §2.2 tab semantics: a tab advances + `col` to the next multiple of 4. + -/ + col : Nat + +namespace Cursor + +/-- Constructs a cursor at the start of `line` (column 0). -/ +@[inline] def ofLine (line : Substring.Raw) : Cursor := + { rest := line, col := 0 } + +/-- The byte position of the cursor. -/ +@[inline] def byteIdx (c : Cursor) : String.Pos.Raw := c.rest.startPos + +/-- The end-of-line byte position. -/ +@[inline] def stopPos (c : Cursor) : String.Pos.Raw := c.rest.stopPos + +/-- The source string. -/ +@[inline] def str (c : Cursor) : String := c.rest.str + +/-- The `Syntax.Range` from the cursor to end-of-line. -/ +@[inline] def toEol (c : Cursor) : Syntax.Range := + { start := c.rest.startPos, stop := c.rest.stopPos } + +/-- +Advances the cursor to byte position `p` and column `newCol`. The caller is responsible for ensuring +`c.byteIdx ≤ p ≤ c.stopPos` and that `newCol` matches the actual column at `p`. +-/ +@[inline] def moveTo (c : Cursor) (p : String.Pos.Raw) (newCol : Nat) : Cursor := + { rest := { c.rest with startPos := p }, col := newCol } + +/-- +Peeks the character at the cursor, returning it together with the byte position of the *next* +character. Returns `none` when the cursor is at or past `stopPos` (i.e., at end-of-line for a line +cursor). + +For tab-aware advancement use `countIndentCols` or `consumeIndentCapped`. +-/ +@[inline] def peek? (c : Cursor) : Option (Char × String.Pos.Raw) := + if c.rest.isEmpty then none else some (c.rest.front, (c.rest.drop 1).startPos) + +/-- +Advances the cursor over a maximal run of characters satisfying `pred`, returning the post-run +cursor and the number of characters consumed. + +The column is incremented by the consumed count, which is correct only when `pred` rejects the tab +character (each matched char then advances the column by exactly one). For tab-aware whitespace +skipping use `countIndentCols` or `consumeIndentCapped`. +-/ +@[inline] def advanceWhile (c : Cursor) (pred : Char → Bool) : Cursor × Nat := Id.run do + let mut rest := c.rest + let mut count := 0 + while !rest.isEmpty do + let ch := rest.front + if !pred ch then break + count := count + 1 + rest := rest.drop 1 + return ({ rest, col := c.col + count }, count) diff --git a/src/Lean/Server/FileWorker/Markdown/Parser/Emphasis.lean b/src/Lean/Server/FileWorker/Markdown/Parser/Emphasis.lean new file mode 100644 index 000000000000..2af19d6d66ab --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser/Emphasis.lean @@ -0,0 +1,286 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +import Init.While +import Lean.Syntax +public import Lean.Server.FileWorker.Markdown.Basic + +namespace Lean.Server.FileWorker.Markdown + +/-! +## Emphasis matching (CommonMark §6.2) + +Implements CommonMark's emit-as-you-go delimiter stack algorithm. + +The single public entry point is `processEmphasis`, which consumes a flat sequence of `InlineTok`s. +Each is either a fully-finalized non-emphasis `Inline` or a raw `*`/`_` delimiter run. It returns +the final `Array Inline` with all matched emphasis spans wrapped as `.bold` or `.italic`. Tokens of +the input are produced by the tokenizer in `Markdown.Parser.Inline`; this module is the second half +of the inline element parsing pass. +-/ + +/-- +A run of `*` or `_` delimiter characters, classified by the CommonMark left/right-flanking rules. +-/ +public structure DelimRun where + /-- Position of the first delimiter character. -/ + startPos : String.Pos.Raw + /-- Position one past the last delimiter character. -/ + endPos : String.Pos.Raw + /-- The delimiter character (`*` or `_`). -/ + ch : Char + /-- Number of consecutive delimiter characters. -/ + origLen : Nat + /-- Whether this run can open emphasis. -/ + canOpen : Bool + /-- Whether this run can close emphasis. -/ + canClose : Bool +deriving Inhabited + +/-- +An intermediate token: either a finalized non-emphasis inline, or a raw emphasis delimiter run +awaiting matching. +-/ +public inductive InlineTok where + | inline (i : Inline) + | delim (run : DelimRun) +deriving Inhabited + +/-- +Appends a `text` inline as an `InlineTok` covering `[textStart:pos)` to `out`, +if non-empty. +-/ +public def pushTokText (out : Array InlineTok) (textStart pos : String.Pos.Raw) : + Array InlineTok := + if textStart < pos then + out.push (.inline (.text { start := textStart, stop := pos })) + else out + +/-- +An open emphasis context on the processing stack that contains a still-active opening delimiter run +with the inline content accumulated since it was pushed. +-/ +structure OpenerFrame where + ch : Char + /-- Position of the leftmost still-available delimiter character. -/ + startPos : String.Pos.Raw + /-- Original total length of the opening run. -/ + origLen : Nat + /-- + The number of delimiter characters not yet consumed. The available chars are the leftmost + `remaining` chars; matches consume from the right. + -/ + remaining : Nat + /-- Whether the original run could close. -/ + canClose : Bool + /-- Inlines accumulated since this opener was pushed. -/ + acc : Array Inline +deriving Inhabited + +/-! +### Delimiter matching + +CommonMark §6.2 specifies emphasis matching as an algorithm over a *delimiter stack*: each `*`/`_` +run that could open emphasis is pushed on the stack while non-delimiter inlines are added to the +inline output, and a closing run looks down the stack for a compatible opener to match against. The +spec describes the algorithm imperatively over two pieces of state, the delimiter stack and the +inline output, and those are maintained in a single `DelimiterMatchState` structure. + +This implementation differs slightly from the spec's pseudocode in that it doesn't keep the inlines +emitted *inside* an open emphasis on the output list and re-walk them on close; instead, they are +kept in the opener itself, so when the opener matches we already have its children ready to wrap. +-/ + +/-- +The state maintained while running CommonMark §6.2's delimiter-matching algorithm: the delimiter +stack, plus the inline output emitted at the document scope (i.e., outside any currently-open +emphasis). + +Each currently-open emphasis candidate is stored in `delimiters` and collects its in-progress +children in its `acc` field; only when the opener's run is fully matched (or abandoned) do those +children make it into a wrapping node and flow into the parent scope (the next opener's `acc`, or +`output` if the stack is empty). +-/ +structure DelimiterMatchState where + /-- The delimiter stack: openers whose runs have not yet been matched. -/ + delimiters : Array OpenerFrame + /-- Inlines emitted at document scope, outside any open emphasis. -/ + output : Array Inline + +/-- The empty initial state. -/ +def DelimiterMatchState.empty : DelimiterMatchState := + { delimiters := #[], output := #[] } + +/-- The state monad for §6.2's delimiter-matching algorithm. -/ +abbrev DelimM := StateM DelimiterMatchState + +/-- +Adds a finalized inline to the inline output (CommonMark §6.2: “added to the inline output”). It +goes into the topmost delimiter's `acc` if the stack is non-empty, otherwise into the bottom output. +-/ +def emit (inl : Inline) : DelimM Unit := modify fun st => + if h : st.delimiters.size > 0 then + let top := st.delimiters.back h + { st with delimiters := st.delimiters.pop.push { top with acc := top.acc.push inl } } + else + { st with output := st.output.push inl } + +/-- Adds a batch of finalized inlines to the inline output (see `emit`). -/ +def emitMany (xs : Array Inline) : DelimM Unit := modify fun st => + if h : st.delimiters.size > 0 then + let top := st.delimiters.back h + { st with delimiters := st.delimiters.pop.push { top with acc := top.acc ++ xs } } + else + { st with output := st.output ++ xs } + +/-- +Pushes a new opener onto the delimiter stack (CommonMark §6.2: “push it to the delimiter stack”). +-/ +def openDelimiter (f : OpenerFrame) : DelimM Unit := + modify fun st => { st with delimiters := st.delimiters.push f } + +/-- +Abandons the topmost delimiter: its still-unmatched delimiter characters are re-emitted as literal +text (CommonMark §6.2: “an unmatched delimiter run is treated as text”), and the inlines that +accumulated inside the would-be emphasis are flushed to the parent scope without any emphasis node +around them. The flush is necessary because we stage in-progress children on the opener's `acc` +field; in the spec's pseudocode they would already sit on the output list and abandoning would be a +stack pop alone. + +This is used both when a closer's match leaves unmatched openers above it on the stack and at the +end to drain the stack. +-/ +def abandonOpener : DelimM Unit := do + let st ← get + if h : st.delimiters.size > 0 then + let top := st.delimiters.back h + let asInlines : Array Inline := + if top.remaining > 0 then + let textRange : Syntax.Range := { + start := top.startPos + -- All delimiter characters are single-byte + stop := { byteIdx := top.startPos.byteIdx + top.remaining } + } + #[Inline.text textRange] ++ top.acc + else top.acc + set { st with delimiters := st.delimiters.pop } + emitMany asInlines + +/-- +Wraps the topmost delimiter's accumulated tokens as `.bold` (when `matchLen == 2`) or `.italic` +(when `matchLen == 1`) using `openDelim`/`closeDelim` as its source ranges, and then either drops +the opener or removes the part that was consumed. + +Precondition: the delimiter stack is non-empty and its top is the opener chosen to match this closer +(any unmatched openers above it have already been abandoned). +-/ +def closeMatched (matchLen : Nat) (openDelim closeDelim : Syntax.Range) : + DelimM Unit := do + let opener := (← get).delimiters.back! + let wrapped : Inline := + if matchLen == 2 then .bold openDelim opener.acc closeDelim + else .italic openDelim opener.acc closeDelim + let newRem := opener.remaining - matchLen + if newRem == 0 then + modify fun st => { st with delimiters := st.delimiters.pop } + emit wrapped + else + modify fun st => + { st with + delimiters := + st.delimiters.pop.push { opener with remaining := newRem, acc := #[wrapped] } } + +/-- +Drains any remaining openers as unmatched and returns the final inline output (CommonMark §6.2: +“process any remaining delimiters as text”). +-/ +def finalizeEmphasis : DelimM (Array Inline) := do + repeat + if (← get).delimiters.isEmpty then break + abandonOpener + return (← get).output + +/-- +Scans the delimiter stack top-down for an opener compatible with the closer `run`, honoring +CommonMark §6.2's rule about lengths that are a multiple of 3. Returns the index (from the bottom) +of the first compatible opener, or `none`. +-/ +def findMatchingOpener (stack : Array OpenerFrame) (run : DelimRun) : Option Nat := do + for h : i in [0:stack.size] do + have : stack.size > 0 := Nat.zero_lt_of_lt h.2.1 + let opener := stack[stack.size - (i + 1)]'(Nat.sub_lt this (Nat.zero_lt_succ i)) + if opener.ch == run.ch then + let lenSum := opener.origLen + run.origLen + let bothCanBoth := opener.canClose || run.canOpen + let blocked := + bothCanBoth && lenSum % 3 == 0 && + (opener.origLen % 3 != 0 || run.origLen % 3 != 0) + unless blocked do + return stack.size - (i + 1) + failure + +/-- +Processes one delimiter run from the token stream. If possible, it uses it as a closer, repeatedly +matching it against compatible openers on the stack and abandoning any unmatched openers above each +match. If any chars remain in the run, they are pushed if the run can open emphasis, or emitted as +text otherwise. +-/ +def processDelimRun (run : DelimRun) : DelimM Unit := do + let mut runRem := run.origLen + let mut runStart := run.startPos + if run.canClose then + let mut keepTrying := true + while runRem > 0 && keepTrying do + match findMatchingOpener (← get).delimiters run with + | none => keepTrying := false + | some idx => + while (← get).delimiters.size > idx + 1 do + abandonOpener + let opener := (← get).delimiters.back! + let matchLen := if opener.remaining >= 2 && runRem >= 2 then 2 else 1 + let openDelim : Syntax.Range := { + -- All delimiter characters are single-byte, so this arithmetic makes sense + start := { byteIdx := opener.startPos.byteIdx + opener.remaining - matchLen } + stop := { byteIdx := opener.startPos.byteIdx + opener.remaining } + } + let closeDelim : Syntax.Range := { + start := runStart + -- Delimiter characters are all single-byte, so adding matchLen is sensible here + stop := { byteIdx := runStart.byteIdx + matchLen } + } + closeMatched matchLen openDelim closeDelim + runRem := runRem - matchLen + runStart := { byteIdx := runStart.byteIdx + matchLen } + if runRem > 0 then + if run.canOpen then + openDelimiter { + ch := run.ch + startPos := runStart + origLen := runRem + remaining := runRem + canClose := run.canClose + acc := #[] + } + else + -- The remainder length runRem is in characters, but all delimiters in the run are single-byte + emit <| + .text { start := runStart, stop := { byteIdx := runStart.byteIdx + runRem } } + +/-- +Processes the emphasis delimiter runs in `toks`, producing a final flat inline array. Implements +CommonMark §6.2's delimiter-matching algorithm (including the rule about lengths that are a multiple +of 3). Unmatched delimiters are turned into literal text. +-/ +public def processEmphasis (toks : Array InlineTok) : Array Inline := + let go : DelimM (Array Inline) := do + for tok in toks do + match tok with + | .inline inl => emit inl + | .delim run => processDelimRun run + finalizeEmphasis + go.run' .empty diff --git a/src/Lean/Server/FileWorker/Markdown/Parser/Inline.lean b/src/Lean/Server/FileWorker/Markdown/Parser/Inline.lean new file mode 100644 index 000000000000..64ab6680eed9 --- /dev/null +++ b/src/Lean/Server/FileWorker/Markdown/Parser/Inline.lean @@ -0,0 +1,715 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: David Thrane Christiansen +-/ +module + +prelude +public import Init.Data.String.PosRaw +public import Init.Data.String.TakeDrop +public import Init.Data.Array.Basic +public import Init.While +public import Std.Data.HashMap +public import Lean.Syntax +public import Lean.Server.FileWorker.Markdown.Basic +public import Lean.Server.FileWorker.Markdown.Parser.Block +import Lean.Server.FileWorker.Markdown.Parser.Emphasis + +namespace Lean.Server.FileWorker.Markdown + +/-! +# Pass 2: Inline Parser + +Walks an inline-bearing source range and recovers all Markdown inline elements except embedded HTML. + +A backslash escape (`\X`) is treated as two characters of plain text and disables any markup +interpretation of `X`. +-/ + +/-- Counts consecutive characters equal to `c` at the start of `sub`. -/ +def countRun (sub : InlineRange) (c : Char) : Nat × String.Pos.Raw := Id.run do + let mut sub := sub + let mut n := 0 + while !sub.isEmpty && sub.front == c do + n := n + 1 + sub := sub.drop1 + return (n, sub.startPos) + +/-- Appends a `text` inline covering `[textStart:pos)` to `out`, if non-empty. -/ +def pushText (out : Array Inline) (textStart pos : String.Pos.Raw) : Array Inline := + if textStart < pos then + out.push (.text { start := textStart, stop := pos }) + else out + +/-- +Whether `c` is in CommonMark's ASCII-punctuation set: ``!"#$%&'()*+,-./:;<=>?@[\]^_\`{|}~``. These +are the only characters that a backslash escape consumes; a backslash before any other character is +a literal backslash followed by the character. +-/ +def isAsciiPunctuation (c : Char) : Bool := + let n := c.toNat + (33 ≤ n && n ≤ 47) || -- ! " # $ % & ' ( ) * + , - . / + (58 ≤ n && n ≤ 64) || -- : ; < = > ? @ + (91 ≤ n && n ≤ 96) || -- [ \ ] ^ _ ` + (123 ≤ n && n ≤ 126) -- { | } ~ + +/-- Tries to parse a code span. Caller ensures ``sub.front == '`'``. -/ +def code (sub : InlineRange) : Option (Inline × String.Pos.Raw) := do + let s := sub.source + let stopPos := sub.stopPos + let cursor := sub.startPos + let (openLen, p) := countRun sub '`' + guard (openLen > 0) + let openTicks : Syntax.Range := { start := cursor, stop := p } + let mut rest : InlineRange := s.range p stopPos + while !rest.isEmpty do + if rest.front == '`' then + let q := rest.startPos + let (runLen, qAfterRun) := countRun rest '`' + if runLen == openLen then + let content : Syntax.Range := { start := p, stop := q } + let closeTicks : Syntax.Range := { start := q, stop := qAfterRun } + return (.code openTicks content closeTicks, qAfterRun) + rest := s.range qAfterRun stopPos + else + rest := rest.drop1 + failure + +/-- +Whether `c` is in the email autolink local-part character set (CommonMark §6.5): ASCII alphanumerics +plus ``.!#$%&'*+/=?^_`{|}~-``. +-/ +def isEmailLocalChar (c : Char) : Bool := + c.isAlpha || c.isDigit || c == '.' || c == '!' || c == '#' + || c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' + || c == '+' || c == '/' || c == '=' || c == '?' || c == '^' + || c == '_' || c == '`' || c == '{' || c == '|' || c == '}' + || c == '~' || c == '-' + +/-- +Tries to parse a URI autolink starting at `start`, which should be the position immediately after +the opening `<`. Returns the position of the closing `>` on success. +-/ +def autolinkUri (sub : InlineRange) : Option String.Pos.Raw := do + let mut rest := sub + guard <| !rest.isEmpty && rest.front.isAlpha + rest := rest.drop1 + let mut schemeLen := 1 + while !rest.isEmpty && schemeLen < 32 do + let c := rest.front + if c.isAlpha || c.isDigit || c == '+' || c == '-' || c == '.' then + schemeLen := schemeLen + 1 + rest := rest.drop1 + else break + guard <| schemeLen ≥ 2 + guard <| !rest.isEmpty && rest.front == ':' + rest := rest.drop1 + while !rest.isEmpty do + let c := rest.front + if c == '>' then return rest.startPos + guard !(c == '<' || c == ' ' || c == '\t' || c == '\n' || c == '\r') + -- Reject ASCII control characters (0x7f is DEL, the others are below 0x20) + guard <| c.toNat ≥ 0x20 && c.toNat != 0x7f + rest := rest.drop1 + failure + +/-- +Tries to parse an email autolink starting at `start`, which is the position immediately after the +opening `<`. Matches CommonMark §6.5's HTML5-style email regex. Returns the position of the closing +`>` on success. +-/ +def autolinkEmail (sub : InlineRange) : Option String.Pos.Raw := do + let s := sub.source + let mut rest := sub + let mut localLen := 0 + while !rest.isEmpty do + let c := rest.front + if isEmailLocalChar c then + localLen := localLen + 1 + rest := rest.drop1 + else break + guard <| localLen != 0 + guard <| !rest.isEmpty && rest.front == '@' + rest := rest.drop1 + -- Domain: at least one label. + while !rest.isEmpty do + let firstC := rest.front + guard firstC.isAlphanum + let labelStart := rest.startPos + rest := rest.drop1 + let mut labelLen := 1 + while !rest.isEmpty && labelLen < 63 do + let c := rest.front + if c.isAlpha || c.isDigit || c == '-' then + labelLen := labelLen + 1 + rest := rest.drop1 + else break + -- A label cannot end with a hyphen. + let labelEndPrev : String.Pos.Raw := { byteIdx := rest.startPos.byteIdx - 1 } + guard <| (labelEndPrev < labelStart) || labelEndPrev.get s.str != '-' + guard !rest.isEmpty + let c := rest.front + if c == '>' then + return rest.startPos + if c == '.' then + rest := rest.drop1 + else + failure + failure + +/-- +Tries to parse an autolink at the beginning of `sub` (which must point at `<`). Returns the resulting +`Inline.autolink` and the position past the closing `>` on success. +-/ +def autolink (sub : InlineRange) : Option (Inline × String.Pos.Raw) := do + let s := sub.source + let stop := sub.stopPos + let cursor := sub.startPos + guard <| !sub.isEmpty && sub.front == '<' + let afterOpen := sub.drop1 + let inner := afterOpen.startPos + let openAngle : Syntax.Range := { start := cursor, stop := inner } + let (isLink, closePos) ← ((true, ·) <$> autolinkUri afterOpen) <|> ((false, ·) <$> autolinkEmail afterOpen) + let closeRest : InlineRange := s.range closePos stop + if closeRest.isEmpty then failure + let afterClose := (closeRest.drop1).startPos + let urlRange : Syntax.Range := { start := inner, stop := closePos } + let closeAngle : Syntax.Range := { start := closePos, stop := afterClose } + if isLink then + return (.autolink openAngle urlRange closeAngle false, afterClose) + else + return (.autolink openAngle urlRange closeAngle true, afterClose) + +/-- +Scans `[…]` (or `![…]`'s alt) for the matching `]`, respecting Markdown's complex error-recovery +rules. Returns the position of the matching `]` if found. +-/ +def findCloseBracket (sub : InlineRange) : Option String.Pos.Raw := do + let s := sub.source + let stop := sub.stopPos + let mut rest := sub + let mut depth : Nat := 0 + while !rest.isEmpty do + let c := rest.front + let p := rest.startPos + let restNext := rest.drop1 + if c == '\\' && !restNext.isEmpty then + rest := restNext.drop1 + else if c == '`' then + -- Code spans bind tighter than link brackets. If a span forms, + -- advance past it; otherwise the entire opening run is literal + -- (CommonMark §6.1 — see the matching scanner branch). + match code rest with + | some (_, after) => rest := s.range after stop + | none => + let (_, runEnd) := countRun rest '`' + rest := s.range runEnd stop + else if c == '<' then + -- Autolinks (and raw HTML, which we don't support) bind tighter + -- than link brackets per CommonMark §6.3: a `]` inside an + -- autolink is part of its URL, not a link close. + match autolink rest with + | some (_, after) => rest := s.range after stop + | none => rest := restNext + else if c == '[' then + depth := depth + 1 + rest := restNext + else if c == ']' then + if depth == 0 then return p + else depth := depth - 1 + rest := restNext + else + rest := restNext + failure + +/-- Scans for `)` at or after `sub.startPos`, respecting backslash escapes. -/ +def findCloseParen (sub : InlineRange) : Option String.Pos.Raw := do + let mut rest := sub + while !rest.isEmpty do + let c := rest.front + let p := rest.startPos + let restNext := rest.drop1 + if c == '\\' && !restNext.isEmpty then + rest := restNext.drop1 + else if c == ')' then + return p + else + rest := restNext + failure + +/-- Skips ASCII whitespace (' ', '\t', '\r', and '\n') at the start of `sub`. -/ +def skipInlineWs (sub : InlineRange) : String.Pos.Raw := Id.run do + let mut sub := sub + while !sub.isEmpty do + let c := sub.front + if c == ' ' || c == '\t' || c == '\n' || c == '\r' then + sub := sub.drop1 + else break + return sub.startPos + +/-- +Parses the body of an inline link/image target, which is the contents between the opening `(` and +its matching `)`. `sub` should begin just after the opening `(`. + + The returned `url` range covers only the URL content (interior of the angle brackets, when +bracketed). Returns `(url, title?, closeParenPos)` on success. +-/ +def inlineLinkTarget (sub : InlineRange) : + Option (Syntax.Range × Option Syntax.Range × String.Pos.Raw) := do + let s := sub.source + let stop := sub.stopPos + let p0 := skipInlineWs sub + let rest : InlineRange := s.range p0 stop + let (url, afterUrlByte) ← + if rest.isEmpty then + pure ({ start := p0, stop := p0 }, p0) + else if rest.front == '<' then + bracketedUrl rest + else + unbracketedUrl rest + let afterUrl := skipInlineWs (s.range afterUrlByte stop) + let afterUrlRest : InlineRange := s.range afterUrl stop + if afterUrlRest.isEmpty then failure + let cAfter := afterUrlRest.front + let mut title? : Option Syntax.Range := none + let mut afterTitle := afterUrl + if cAfter == '"' || cAfter == '\'' || cAfter == '(' then + let closer : Char := if cAfter == '(' then ')' else cAfter + let titleStart := (afterUrlRest.drop1).startPos + let mut titleRest : InlineRange := s.range titleStart stop + let mut titleEnd? : Option String.Pos.Raw := none + while !titleRest.isEmpty do + let c := titleRest.front + let q := titleRest.startPos + let titleRestNext := titleRest.drop1 + if c == '\\' && !titleRestNext.isEmpty then + titleRest := titleRestNext.drop1 + else if c == closer then + titleEnd? := some q + break + else + titleRest := titleRestNext + match titleEnd? with + | none => failure + | some te => + let teRest : InlineRange := s.range te stop + if teRest.isEmpty then failure + let afterTe := (teRest.drop1).startPos + title? := some { start := titleStart, stop := te } + afterTitle := skipInlineWs (s.range afterTe stop) + let afterTitleRest : InlineRange := s.range afterTitle stop + if afterTitleRest.isEmpty then failure + if afterTitleRest.front != ')' then failure + return (url, title?, afterTitle) +where + /-- + Bracketed form ``. The URL content excludes the angle brackets themselves; backslash escapes + are allowed; an unescaped `<` or `\n` inside fails the whole target. Returns the URL range and the + position one past the closing `>`. + -/ + -- Note: deliberately not shared with `autolinkUri` despite the surface similarity. CommonMark + -- §6.3 (link destinations) recognizes backslash escapes inside `<…>` and accepts whitespace; §6.5 + -- (autolinks) does neither and additionally rejects ASCII control chars. A shared loop would need + -- to be parameterised on escape mode and character class, which obscures both call sites more + -- than it deduplicates them. + bracketedUrl (rest : InlineRange) : Option (Syntax.Range × String.Pos.Raw) := do + let mut rest := rest.drop1 -- consume opening `<` + let urlContentStart := rest.startPos + let mut urlContentEnd := urlContentStart + let mut afterUrlByte := urlContentStart + let mut foundClose := false + while !rest.isEmpty do + let c := rest.front + let p := rest.startPos + let restNext := rest.drop1 + if c == '\\' && !restNext.isEmpty then + rest := restNext.drop1 + else if c == '>' then + urlContentEnd := p + afterUrlByte := restNext.startPos + foundClose := true + break + else if c == '<' || c == '\n' then + failure + else + rest := restNext + guard foundClose + return ({ start := urlContentStart, stop := urlContentEnd }, afterUrlByte) + /-- + Unbracketed form. Stops at whitespace, an ASCII control char, or an unbalanced `)`; balances + `(`/`)` runs and rejects targets with unmatched opens (CommonMark §6.3: a link destination's + parens must balance). Returns the URL range and the (same) position right after it. + -/ + unbracketedUrl (rest : InlineRange) : Option (Syntax.Range × String.Pos.Raw) := do + let urlContentStart := rest.startPos + let mut rest := rest + let mut parenDepth : Nat := 0 + while !rest.isEmpty do + let c := rest.front + let restNext := rest.drop1 + if c == '\\' && !restNext.isEmpty then + rest := restNext.drop1 + else if c == ' ' || c == '\t' || c == '\n' || c == '\r' then + break + else if c == '(' then + parenDepth := parenDepth + 1 + rest := restNext + else if c == ')' then + if parenDepth == 0 then break + parenDepth := parenDepth - 1 + rest := restNext + else if c.toNat < 0x20 then + break + else + rest := restNext + guard (parenDepth == 0) + let urlContentEnd := rest.startPos + return ({ start := urlContentStart, stop := urlContentEnd }, urlContentEnd) + +/-- +Classifies a `*`/`_` delimiter run by the characters that surrounding it, determining whether it can +open or close emphasis. `prevChar`/`nextChar` are the chars immediately before/after the run, with +ASCII space substituted at range boundaries. + +Returns a tuple in which the first projection is true when the delimiter run can open emphasis and +the second is true when it can close emphasis. +-/ +def classifyDelim (ch prevChar nextChar : Char) : Bool × Bool := + let beforeWs := prevChar.isWhitespace + let afterWs := nextChar.isWhitespace + let beforePunct := isAsciiPunctuation prevChar + let afterPunct := isAsciiPunctuation nextChar + let leftFlanking := !afterWs && (!afterPunct || beforeWs || beforePunct) + let rightFlanking := !beforeWs && (!beforePunct || afterWs || afterPunct) + let canOpen := + if ch == '*' then leftFlanking + else leftFlanking && (!rightFlanking || beforePunct) + let canClose := + if ch == '*' then rightFlanking + else rightFlanking && (!leftFlanking || afterPunct) + (canOpen, canClose) + +/-- +Parses the common suffix logic for `[…]…` links and `![…]…` images: the caller has already located +the outer `[…]` (with `openBracket`/`closeBracket` covering the brackets, and `afterClose` one past +the `]`) and parsed its interior as `interior`. This function picks among the inline `(url)`, full +reference `[label]`, collapsed `[]`, and shortcut forms, building the result with `mk` (which closes +over the leading `!` for images). +-/ +def linkSuffix + (refs : RefTable) (s : InlineSource) (stop : String.Pos.Raw) + (openBracket closeBracket : Syntax.Range) (afterClose : String.Pos.Raw) + (interior : Array Inline) + (mk : Syntax.Range → Array Inline → Syntax.Range → LinkTarget → Inline) : + Option (Inline × String.Pos.Raw) := + let afterCloseRest : InlineRange := s.range afterClose stop + -- Inline form `(url)` takes precedence; on failure, fall back to shortcut + -- (the second `[` of a reference form would not match here, so the only + -- other option is shortcut). A second `[` commits to the reference form + -- — a missing close bracket or unresolved label is a hard failure. + if !afterCloseRest.isEmpty && afterCloseRest.front == '(' then + inlineForm afterCloseRest <|> shortcutForm + else if !afterCloseRest.isEmpty && afterCloseRest.front == '[' then + referenceForm afterCloseRest + else + shortcutForm +where + /-- The `[…](url)` / `![…](url "title")` form. -/ + inlineForm (afterCloseRest : InlineRange) : Option (Inline × String.Pos.Raw) := do + let urlStart := (afterCloseRest.drop1).startPos + let openParen : Syntax.Range := { start := afterClose, stop := urlStart } + let (url, title?, parenEnd) ← + inlineLinkTarget (s.range urlStart stop) + let parenEndRest : InlineRange := s.range parenEnd stop + guard !parenEndRest.isEmpty + let afterParen := (parenEndRest.drop1).startPos + let closeParen : Syntax.Range := { start := parenEnd, stop := afterParen } + let target := LinkTarget.inline openParen url closeParen title? + return (mk openBracket interior closeBracket target, afterParen) + /-- The full `[…][label]` / `![…][label]` and collapsed `[…][]` / `![…][]` forms. -/ + referenceForm (afterCloseRest : InlineRange) : Option (Inline × String.Pos.Raw) := do + let labelStart := (afterCloseRest.drop1).startPos + let labelOpen : Syntax.Range := { start := afterClose, stop := labelStart } + let refEnd ← findCloseBracket (s.range labelStart stop) + let refEndRest : InlineRange := s.range refEnd stop + guard !refEndRest.isEmpty + let afterRefEnd := (refEndRest.drop1).startPos + let labelClose : Syntax.Range := { start := refEnd, stop := afterRefEnd } + let isCollapsed := labelStart == refEnd + -- Collapsed reference: the link text/image alt serves as the label. + let labelStr := + if isCollapsed then InlineSource.extract s openBracket.stop closeBracket.start + else InlineSource.extract s labelStart refEnd + let { url, title? } ← refs[normalizeLabel labelStr]? + let form : ReferenceLinkForm := + if isCollapsed then .collapsed labelOpen labelClose + else .full labelOpen { start := labelStart, stop := refEnd } labelClose + return (mk openBracket interior closeBracket (.reference url title? form), afterRefEnd) + /-- The shortcut `[…]` / `![…]` form: the bracket interior itself is the label. -/ + shortcutForm : Option (Inline × String.Pos.Raw) := do + let labelStr := InlineSource.extract s openBracket.stop closeBracket.start + let { url, title? } ← refs[normalizeLabel labelStr]? + return (mk openBracket interior closeBracket (.reference url title? .shortcut), afterClose) + +/-! +### Inline Tokenization + +The §6.2 emphasis algorithm consumes a flat sequence of tokens, each either a fully-finalized +non-emphasis `Inline` (code span, link, image, autolink, soft/hard break, escaped char) or a +still-raw `*`/`_` delimiter run. `tokenizeInlines` produces this sequence. It walks the inline +source, recognizes each non-emphasis inline kind in turn (backslash escape §2.4, code span §6.1, +link §6.3, image §6.4, autolink §6.5, hard line break §6.7), and saves delimiter runs for §6.2 to +handle later. + +The walker's mutable state (`InlineTokenizerState` below) is touched through three operations: +`recognize` for “matched an inline,” `skipChar` for “no match, this character is literal text,” and +`skipPast` for the one place where we leap forward past several literal chars at once. +-/ + +/-- +The state maintained while walking an inline source and emitting tokens +for §6.2: the tokens emitted so far, the unconsumed input, the start of +any pending plain-text run not yet flushed, and the previous source +character (needed for §6.2's flanking-rule classification of `*`/`_` +runs as openers/closers). + +A pending plain-text run is the consecutive characters seen since the +last token boundary that did *not* start any recognized inline; these +are buffered and flushed as a single `.text` inline immediately before +the next emitted token (or at end of input). +-/ +structure InlineTokenizerState where + /-- Tokens emitted so far. -/ + out : Array InlineTok + /-- The unconsumed input. -/ + rest : InlineRange + /-- Start of the pending plain-text run not yet flushed. -/ + textStart : String.Pos.Raw + /-- + The source character immediately before `rest.startPos`. Used to classify emphasis delimiter runs + as left or right flanking. + -/ + prevChar : Char + +/-- The empty initial state covering `s[start:stop)`. -/ +def InlineTokenizerState.init (s : InlineSource) (start stop : String.Pos.Raw) : + InlineTokenizerState where + out := #[] + rest := s.range start stop + textStart := start + prevChar := ' ' + +/-- The state monad for inline tokenization. -/ +abbrev TokenizeM := StateM InlineTokenizerState + +/-- +Handles a recognized inline: + 1. Flushes any pending plain-text run from `textStart` up to `matchStart`. + 2. Appends `tok`. + 3. Advances the walker to `resumeAt`, recording `newPrev` as the post-match flanking character. + +Most callers pass `matchStart := cursor` (the position the match began), so all consumed bytes are +emitted exactly once. The line-ending branch passes a *trimmed* position to drop trailing spaces. +-/ +def recognize (tok : InlineTok) (matchStart resumeAt : String.Pos.Raw) + (resumeRest : InlineRange) (newPrev : Char) : TokenizeM Unit := + modify fun st => + { st with + out := (pushTokText st.out st.textStart matchStart).push tok, + rest := resumeRest, + textStart := resumeAt, + prevChar := newPrev + } + +/-- +The current character did not start a recognized inline. Advances one character, accumulating it +into the pending plain-text run. +-/ +def skipChar : TokenizeM Unit := modify fun st => + { st with rest := st.rest.drop1, prevChar := st.rest.front } + +/-- +Advances multiple characters as plain text. + +This is used when a code span is not properly closed, forcing its opener to become text instead. +-/ +def skipPast (newRest : InlineRange) (newPrev : Char) : TokenizeM Unit := + modify fun st => { st with rest := newRest, prevChar := newPrev } + +/-- Flushes residual pending text and returns the final token array. -/ +def finalizeTokens : TokenizeM (Array InlineTok) := do + let st ← get + return pushTokText st.out st.textStart st.rest.startPos + +mutual + +/-- +Parses the inline content of a substring into a flat array of inlines. +-/ +partial def parseInlinesInRange (refs : RefTable) (s : InlineSource) (start stop : String.Pos.Raw) : + Array Inline := + processEmphasis (tokenizeInlines refs s start stop) + +/-- +Tokenizes the inline content of a block into a flat sequence of already-finalized non-emphasis +inlines and unprocessed emphasis delimiter runs. Emphasis matching is deferred to `processEmphasis`. +-/ +partial def tokenizeInlines (refs : RefTable) (s : InlineSource) (start stop : String.Pos.Raw) : + Array InlineTok := + let go : TokenizeM (Array InlineTok) := do + while !(← get).rest.isEmpty do + let st ← get + let c := st.rest.front + let cursor := st.rest.startPos + let restNext := st.rest.drop1 + let cursorNext := restNext.startPos + if c == '\\' && !restNext.isEmpty then + let nextC := restNext.front + let restAfterEsc := restNext.drop1 + let escapedNext := restAfterEsc.startPos + if nextC == '\n' then + -- Backslash before a line ending is a hard line break (§6.7). + recognize (.inline (.hardBreak { start := cursor, stop := cursorNext })) + cursor escapedNext restAfterEsc ' ' + else if isAsciiPunctuation nextC then + recognize (.inline (.text { start := cursorNext, stop := escapedNext })) + cursor escapedNext restAfterEsc nextC + else + skipChar + else if c == '`' then + match code st.rest with + | some (codeSpan, after) => + recognize (.inline codeSpan) cursor after (s.range after stop) '`' + | none => + let (_, runEnd) := countRun st.rest '`' + skipPast (s.range runEnd stop) '`' + else if c == '!' && !restNext.isEmpty && restNext.front == '[' then + match image refs s cursor stop with + | some (img, after) => + recognize (.inline img) cursor after (s.range after stop) ')' + | none => skipChar + else if c == '[' then + match link refs s cursor stop with + | some (link, after) => + recognize (.inline link) cursor after (s.range after stop) ')' + | none => skipChar + else if c == '<' then + match autolink st.rest with + | some (autolink, after) => + recognize (.inline autolink) cursor after (s.range after stop) '>' + | none => skipChar + else if c == '\n' then + -- A line ending normalizes to a soft break (or, with two or more trailing spaces, a hard + -- break — §6.7). Per §4.8 paragraph-content normalization, trailing whitespace before the + -- line ending is dropped in either case so it doesn't leak into the rendered output. + let mut q := cursor + while q > st.textStart && (String.Pos.Raw.prev s.str q).get s.str == ' ' do + q := String.Pos.Raw.prev s.str q + let spaces := cursor.byteIdx - q.byteIdx + -- The emitted token's stop must be one byte past the `\n` itself; `cursorNext` skips across + -- the inter-line gap to the next line's first non-whitespace position (correct for the + -- walker but would make the token range cover the gap). + let stopPastNewline : String.Pos.Raw := ⟨cursor.byteIdx + 1⟩ + if spaces >= 2 then + recognize (.inline (.hardBreak { start := q, stop := stopPastNewline })) + q cursorNext restNext ' ' + else + -- Soft break: emit the bare `\n` as a standalone text inline so the trimmed text and any + -- following text stay separate (and the renderer's soft break rendering remains a single + -- `\n`). + recognize (.inline (.text { start := cursor, stop := stopPastNewline })) + q cursorNext restNext '\n' + else if c == '*' || c == '_' then + let (n, runEnd) := countRun st.rest c + let runEndRest : InlineRange := s.range runEnd stop + let nextC := if !runEndRest.isEmpty then runEndRest.front else ' ' + let (canOpen, canClose) := classifyDelim c st.prevChar nextC + recognize + (.delim { startPos := cursor, endPos := runEnd, ch := c, origLen := n, canOpen, canClose }) + cursor runEnd runEndRest c + else + skipChar + finalizeTokens + go.run' (.init s start stop) + +/-- +Whether `inlines` contains a link or autolink element, recursively descending into emphasis but +*not* into images. Per CommonMark §6.3, a link's text may not contain another link at any level of +nesting. If it does, the outer link is suppressed and its bracket characters become literal text. +Images are excluded because the spec explicitly permits images inside link text, and links inside +those images' alt text. +-/ +partial def containsLink (inlines : Array Inline) : Bool := + inlines.any fun i => match i with + | .link .. => true + | .autolink .. => true + | .italic _ content _ => containsLink content + | .bold _ content _ => containsLink content + | _ => false + +/-- +Locates the outer `[…]` of a link or image at `cursor`, parses its interior, and hands off to +`parseLinkSuffix` to decide which `[…]…` form applies. `prefixLen` is the number of characters +preceding the opening `[` consumed at `cursor` (0 for links, 1 for `!` of images). + +Returns `none` when no `[…]` is present, when the closing bracket is missing, or when `reject?` +rejects the parsed interior. `reject?` implements CommonMark §6.3's “links may not contain other +links” rule for links. Images pass `none`. +-/ +partial def bracketed + (refs : RefTable) (s : InlineSource) (cursor stop : String.Pos.Raw) + (prefixLen : Nat) + (reject? : Option (Array Inline → Bool)) + (mk : Syntax.Range → Array Inline → Syntax.Range → LinkTarget → Inline) : + Option (Inline × String.Pos.Raw) := do + let cursorRest : InlineRange := s.range cursor stop + -- Step over the (possibly empty) prefix to reach the `[`. + let mut p := cursorRest + for _ in [0 : prefixLen] do + guard !p.isEmpty + p := p.drop1 + guard <| !p.isEmpty && p.front == '[' + let bracketStart := p.startPos + let interiorStart := (p.drop1).startPos + let openBracket : Syntax.Range := { start := bracketStart, stop := interiorStart } + let bracketEnd ← findCloseBracket (s.range interiorStart stop) + + let bracketEndRest : InlineRange := s.range bracketEnd stop + guard !bracketEndRest.isEmpty + let afterClose := (bracketEndRest.drop1).startPos + let closeBracket : Syntax.Range := { start := bracketEnd, stop := afterClose } + let interior := parseInlinesInRange refs s interiorStart bracketEnd + if let some r := reject? then + guard !(r interior) + linkSuffix refs s stop openBracket closeBracket afterClose interior mk + +/-- Tries to parse a link `[text](url)`, `[text][ref]`, `[text][]`, or `[text]`. -/ +partial def link (refs : RefTable) (s : InlineSource) (cursor stop : String.Pos.Raw) : + Option (Inline × String.Pos.Raw) := + -- CommonMark §6.3: a link's text may not contain another link at any + -- level of nesting. If it does, the outer link is not formed: its + -- bracket characters fall through to literal text and the inner link + -- survives. + bracketed refs s cursor stop 0 (some containsLink) .link + +/-- +Tries to parse `![alt](url)`, `![alt][ref]`, `![alt][]`, or `![alt]`. +-/ +partial def image (refs : RefTable) (s : InlineSource) (cursor stop : String.Pos.Raw) : + Option (Inline × String.Pos.Raw) := do + -- Bang range covers just the `!`; `tryParseBracketed` skips it via + -- `prefixLen := 1` and finds the `[` that follows. + let cursorRest : InlineRange := s.range cursor stop + guard !cursorRest.isEmpty + let bang : Syntax.Range := { start := cursor, stop := (cursorRest.drop1).startPos } + bracketed refs s cursor stop 1 none (.image bang) + +end + +/-- +Parses the inline content of a single source range into a flat array of inlines. The range is +typically a paragraph's full extent or a heading's content slice. +-/ +public def parseInlines (refs : RefTable) (s : InlineSource) (range : Syntax.Range) : Array Inline := + parseInlinesInRange refs s range.start range.stop diff --git a/src/Lean/Server/FileWorker/SemanticHighlighting.lean b/src/Lean/Server/FileWorker/SemanticHighlighting.lean index 1d84c46700b5..5f78e045a3c1 100644 --- a/src/Lean/Server/FileWorker/SemanticHighlighting.lean +++ b/src/Lean/Server/FileWorker/SemanticHighlighting.lean @@ -7,6 +7,7 @@ module prelude public import Lean.Server.Requests +public import Lean.Server.FileWorker.Markdown public section @@ -225,7 +226,6 @@ private def HandleOverlapState.token (st : HandleOverlapState) (t : AbsoluteLspS return st - /-- Eliminates overlapping tokens by selecting a single “best” token for each interval between token boundaries. @@ -328,13 +328,273 @@ private def splitStr (text : FileMap) (stx : Syntax) : Array Syntax := Id.run do return stxs +/-! ## Markdown highlighting -open Lean.Doc.Syntax in +Walks the result of the full Markdown parse (`Lean.Server.FileWorker.Markdown.parseDocument`, +which runs both passes) and emits `LeanSemanticToken`s. The walk is purely structural — no +inline parsing happens here; each `Block (Array Inline)` already carries its parsed inlines +with file-anchored positions. +-/ + +namespace Markdown + +/-- Block context for combining markup highlighting types. -/ +private inductive MarkupBlockCtxt where + | none + | heading + | quote + | list + +/-- +Active markup attributes during the walk: which emphasis is in scope and the +nearest enclosing block context. `markupTypeOf?` maps an attribute set to the +matching `SemanticTokenType` constructor (or `none` for plain text in plain +context, which is not emitted as a markup token). +-/ +private structure MarkupAttrs where + bold : Bool := false + italic : Bool := false + block : MarkupBlockCtxt := .none + +@[inline] private def MarkupAttrs.withBold (a : MarkupAttrs) : MarkupAttrs := + { a with bold := true } + +@[inline] private def MarkupAttrs.withItalic (a : MarkupAttrs) : MarkupAttrs := + { a with italic := true } + +@[inline] private def MarkupAttrs.withBlock (a : MarkupAttrs) (b : MarkupBlockCtxt) : MarkupAttrs := + { a with block := b } + +private def markupTypeOf : MarkupAttrs → SemanticTokenType + | { bold := false, italic := false, block := .none } => .markupDocText + | { bold := true, italic := false, block := .none } => .markupBold + | { bold := false, italic := true, block := .none } => .markupItalic + | { bold := true, italic := true, block := .none } => .markupBoldItalic + | { bold := false, italic := false, block := .heading } => .markupHeading + | { bold := true, italic := false, block := .heading } => .markupBoldHeading + | { bold := false, italic := true, block := .heading } => .markupItalicHeading + | { bold := true, italic := true, block := .heading } => .markupBoldItalicHeading + | { bold := false, italic := false, block := .quote } => .markupQuote + | { bold := true, italic := false, block := .quote } => .markupBoldQuote + | { bold := false, italic := true, block := .quote } => .markupItalicQuote + | { bold := true, italic := true, block := .quote } => .markupBoldItalicQuote + | { bold := false, italic := false, block := .list } => .markupList + | { bold := true, italic := false, block := .list } => .markupBoldList + | { bold := false, italic := true, block := .list } => .markupItalicList + | { bold := true, italic := true, block := .list } => .markupBoldItalicList + +/-- +Converts a `Syntax.Range` to a `Syntax` carrying the same source span. +The resulting syntax is suitable as a `LeanSemanticToken.stx`; the LSP layer +already handles raw byte offsets via `text.utf8PosToLspPos`. +-/ +@[inline] private def srcRangeToSyntax (r : Syntax.Range) : Syntax := + Syntax.ofRange ⟨r.start, r.stop⟩ + +/-- +Pushes one `LeanSemanticToken` per source line covered by `r`. Single-line ranges go through +unchanged; multi-line ranges (e.g. fenced code blocks) are split at newline boundaries. +-/ +private def pushSplit (text : FileMap) (out : Array LeanSemanticToken) + (r : Syntax.Range) (tokenType : SemanticTokenType) (priority : Nat := 5) : + Array LeanSemanticToken := Id.run do + if r.start == r.stop then return out + let stx := srcRangeToSyntax r + let mut out := out + for line in splitStr text stx do + out := out.push { stx := line, type := tokenType, priority } + return out + +/-- +Pushes a single-line `LeanSemanticToken` directly without going through `splitStr`. The caller +guarantees `r` does not span a newline. +-/ +@[inline] private def pushTok (out : Array LeanSemanticToken) (r : Syntax.Range) + (tokenType : SemanticTokenType) (priority : Nat := 5) : Array LeanSemanticToken := + if r.start == r.stop then out + else out.push { stx := srcRangeToSyntax r, type := tokenType, priority } + +private partial def emitInlines (text : FileMap) (a : MarkupAttrs) + (inlines : Array Markdown.Inline) (out : Array LeanSemanticToken) : + Array LeanSemanticToken := Id.run do + let mut out := out + for i in inlines do + out := emitInline text a i out + return out +where + emitInline (text : FileMap) (a : MarkupAttrs) (i : Markdown.Inline) + (out : Array LeanSemanticToken) : Array LeanSemanticToken := Id.run do + let mut out := out + match i with + | .text r => + -- A soft break emits a one-byte `.text` covering just the `\n`. The + -- inline tree retains it so downstream consumers (e.g. an HTML + -- renderer) can reproduce the line ending, but for LSP semantic + -- tokens it would translate to a multi-line token, which VS Code + -- doesn't support. + let isNewlineOnly := r.start.byteIdx + 1 == r.stop.byteIdx + && r.start.get text.source == '\n' + unless isNewlineOnly do + out := pushTok out r (markupTypeOf a) (priority := 4) + | .code openTicks content closeTicks => + out := pushTok out openTicks .keyword + out := pushTok out closeTicks .keyword + out := pushSplit text out content .markupInlineCode (priority := 6) + | .italic openDelim content closeDelim => + out := pushTok out openDelim .keyword + out := pushTok out closeDelim .keyword + out := emitInlines text a.withItalic content out + | .bold openDelim content closeDelim => + out := pushTok out openDelim .keyword + out := pushTok out closeDelim .keyword + out := emitInlines text a.withBold content out + | .link openBracket inner closeBracket target => + out := emitBracketed none openBracket inner closeBracket target out + | .image bang openBracket alt closeBracket target => + out := emitBracketed (some bang) openBracket alt closeBracket target out + | .hardBreak _ => pure () + | .autolink openAngle url closeAngle _ => + out := pushTok out openAngle .keyword + out := pushTok out closeAngle .keyword + out := pushTok out url .markupUrl + return out + + /-- + Emits the structural tokens for a reference-style link's bracket pair(s). + + For full form, the link's text content is walked as ordinary inlines and the second `[label]` + brackets get keyword/cross-ref tokens. For collapsed and shortcut form, the link text *is* the + label, so `markupCrossReference` covers the bracket interior in place of the inline walk. In + collapsed form, the empty `[]` is annotated as a keyword. + -/ + emitReferenceForm (text : FileMap) (a : MarkupAttrs) + (out : Array LeanSemanticToken) (interior : Syntax.Range) + (inner : Array Markdown.Inline) (form : Markdown.ReferenceLinkForm) : + Array LeanSemanticToken := Id.run do + let mut out := out + match form with + | .full openB label closeB => + out := emitInlines text a inner out + out := pushTok out openB .keyword + out := pushTok out closeB .keyword + out := pushTok out label .markupCrossReference (priority := 6) + | .collapsed openB closeB => + out := pushTok out interior .markupCrossReference (priority := 6) + out := pushTok out openB .keyword + out := pushTok out closeB .keyword + | .shortcut => + out := pushTok out interior .markupCrossReference (priority := 6) + return out + + /-- + Emits the tokens shared by `[…]…` links and `![…]…` images. + -/ + emitBracketed (bang? : Option Syntax.Range) + (openBracket : Syntax.Range) (inner : Array Markdown.Inline) + (closeBracket : Syntax.Range) (target : Markdown.LinkTarget) + (out : Array LeanSemanticToken) : Array LeanSemanticToken := Id.run do + let mut out := out + if let some bang := bang? then + out := pushTok out bang .keyword + out := pushTok out openBracket .keyword + out := pushTok out closeBracket .keyword + let interior : Syntax.Range := + { start := openBracket.stop, stop := closeBracket.start } + match target with + | .inline openParen url closeParen title? => + out := emitInlines text a inner out + out := pushTok out openParen .keyword + out := pushTok out url .markupUrl + if let some t := title? then out := pushTok out t .string + out := pushTok out closeParen .keyword + | .reference _ _ form => + out := emitReferenceForm text a out interior inner form + return out + +/-- +Emits the structural tokens of a matched link reference definition: keyword on the brackets and +colon, cross-reference on the label, URL on the destination, and string on the optional title. The +label and title may span multiple source lines, so they are split at line boundaries. +-/ +private def emitRefDef (text : FileMap) (out : Array LeanSemanticToken) (r : RefDef) : + Array LeanSemanticToken := Id.run do + let mut out := out + out := pushTok out r.openBracket .keyword + out := pushSplit text out r.label .markupCrossReference + out := pushTok out r.closeBracket .keyword + out := pushTok out r.colon .keyword + out := pushTok out r.url .markupUrl + if let some t := r.title? then + out := pushSplit text out t .string + return out + +private partial def emitBlocks (text : FileMap) (a : MarkupAttrs) + (blocks : Array (Markdown.Block (Array Markdown.Inline))) + (out : Array LeanSemanticToken) : Array LeanSemanticToken := Id.run do + let mut out := out + for b in blocks do + out := emitBlock text a b out + return out +where + emitBlock (text : FileMap) (a : MarkupAttrs) + (b : Markdown.Block (Array Markdown.Inline)) + (out : Array LeanSemanticToken) : Array LeanSemanticToken := Id.run do + let mut out := out + match b with + | .paragraph lines => + for inls in lines do + out := emitInlines text a inls out + | .atxHeading hashes content closeHashes? => + out := pushTok out hashes .keyword + if let some c := closeHashes? then out := pushTok out c .keyword + out := emitInlines text (a.withBlock .heading) content out + | .setextHeading _ lines underline => + for inls in lines do + out := emitInlines text (a.withBlock .heading) inls out + out := pushTok out underline .keyword + | .fencedCode info openFence closeFence? lines => + out := pushTok out openFence .keyword + if let some c := closeFence? then out := pushTok out c .keyword + if let some tag := info.infoString? then + out := pushTok out tag .function + for line in lines do + out := pushSplit text out line .markupCodeBlock + | .indentedCode lines => + for (range, _) in lines do + out := pushTok out range .markupCodeBlock + | .blockquote markers children => + for m in markers do + out := pushTok out m .keyword + out := emitBlocks text (a.withBlock .quote) children out + | .list _ _ items => + out := emitBlocks text (a.withBlock .list) items out + | .listItem marker children => + out := pushTok out marker .keyword + out := emitBlocks text (a.withBlock .list) children out + | .linkRefDef m => + out := emitRefDef text out m + | .thematicBreak line => + out := pushTok out line .keyword + return out + +end Markdown + +/-- +Collects semantic tokens for a Markdown docstring. Block delimiters (`#`, `>`, list markers, fence +runs, brackets) are emitted as `.keyword`; URL targets as `.markupUrl`; code block contents as +`.markupCodeBlock`; and inline emphasis as the appropriate `markup*` type combined with the +enclosing block context. Undecorated text is emitted as `.markupDocText`. +-/ +def collectMarkdownTokens (text : FileMap) (startPos endPos : String.Pos.Raw) : + Array LeanSemanticToken := + Markdown.emitBlocks text {} (Markdown.parseDocument text.source startPos endPos) #[] + +open Lean.Doc.Syntax Markdown in private partial def collectVersoTokens (text : FileMap) (stx : Syntax) (getTokens : (stx : Syntax) → Array LeanSemanticToken) : Array LeanSemanticToken := - go stx |>.run #[] |>.2 + go {} stx |>.run #[] |>.2 where tok (tk : Syntax) (k : SemanticTokenType) : StateM (Array LeanSemanticToken) Unit := let priority := @@ -346,7 +606,7 @@ where | _ => 5 modify (·.push { stx := tk, type := k, priority }) - go (stx : Syntax) : StateM (Array LeanSemanticToken) Unit := do + go (a : MarkupAttrs) (stx : Syntax) : StateM (Array LeanSemanticToken) Unit := do match stx with | `(arg_val| $x:ident ) | `(arg_val| $x:str ) @@ -356,12 +616,12 @@ where tok tk1 .keyword tok x .property tok tk2 .keyword - go v + go a v tok tk3 .keyword | `(named_no_paren| $x:ident :=%$tk $v:arg_val ) => tok x .property tok tk .keyword - go v + go a v | `(flag_on| +%$tk$x) | `(flag_off| -%$tk$x) => tok tk .keyword tok x .property @@ -373,21 +633,26 @@ where tok tk1 .keyword tok s .string tok tk2 .keyword - | `(inline|$_:str) | `(inline|line! $_) => pure () -- No tokens for plain text or line breaks - | `(inline| *[%$tk1 $inls* ]%$tk2) | `(inline|_[%$tk1 $inls* ]%$tk2) => + | `(inline|$s:str) => tok s (markupTypeOf a) + | `(inline|line! $_) => pure () -- No token for line breaks + | `(inline| *[%$tk1 $inls* ]%$tk2) => + tok tk1 .keyword + inls.forM (go a.withBold) + tok tk2 .keyword + | `(inline|_[%$tk1 $inls* ]%$tk2) => tok tk1 .keyword - inls.forM go + inls.forM (go a.withItalic) tok tk2 .keyword | `(inline| link[%$tk1 $inls* ]%$tk2 $ref) => tok tk1 .keyword - inls.forM go + inls.forM (go a) tok tk2 .keyword - go ref + go a ref | `(inline| image(%$tk1 $s )%$tk2 $ref) => tok tk1 .keyword tok s .string tok tk2 .keyword - go ref + go a ref | `(inline| footnote(%$tk1 $s )%$tk2) => tok tk1 .keyword tok s .property @@ -399,10 +664,10 @@ where | `(inline| role{%$tk1 $x $args* }%$tk2 [%$tk3 $inls* ]%$tk4) => tok tk1 .keyword tok x .function - args.forM go + args.forM (go a) tok tk2 .keyword tok tk3 .keyword - inls.forM go + inls.forM (go a) tok tk4 .keyword | `(inline| \math%$tk1 code(%$tk2 $s )%$tk3) | `(inline| \displaymath%$tk1 code(%$tk2 $s )%$tk3) => @@ -412,28 +677,28 @@ where tok tk3 .keyword | `(list_item| *%$tk $inls*) => tok tk .keyword - inls.forM go + inls.forM (go (a.withBlock .list)) | `(desc| :%$tk $inls* => $blks*) => tok tk .keyword - inls.forM go - blks.forM go - | `(block|para[$inl*]) => inl.forM go + inls.forM (go (a.withBlock .list)) + blks.forM (go (a.withBlock .list)) + | `(block|para[$inl*]) => inl.forM (go a) | `(block| ```%$tk1 $x $args* | $s ```%$tk2)=> tok tk1 .keyword tok x .function - args.forM go + args.forM (go a) for line in splitStr text s do tok line .string tok tk2 .keyword | `(block| :::%$tk1 $x $args* { $blks* }%$tk2)=> tok tk1 .keyword tok x .function - args.forM go - blks.forM go + args.forM (go a) + blks.forM (go a) tok tk2 .keyword | `(block| command{%$tk1 $x $args*}%$tk2)=> tok tk1 .keyword tok x .function - args.forM go + args.forM (go a) tok tk2 .keyword | `(block| %%%%$tk1 $vals* %%%%$tk2)=> tok tk1 .keyword @@ -448,16 +713,19 @@ where tok tk1 .keyword tok s .property tok tk2 .keyword - inls.forM go + inls.forM (go a) | `(block| header(%$tk $_ ){ $inls* })=> tok tk .keyword - inls.forM go + inls.forM (go (a.withBlock .heading)) + | `(block| >%$tk $blks*) => + tok tk .keyword + blks.forM (go (a.withBlock .quote)) | `(block|ul{$items*}) | `(block|ol($_){$items*}) | `(block|dl{$items*}) => - items.forM go + items.forM (go a) | other => let k := other.getKind if k == nullKind || k == ``Lean.Parser.Command.versoCommentBody then - other.getArgs.forM go + other.getArgs.forM (go a) /-- Collects all semantic tokens that can be deduced purely from `Syntax` @@ -474,9 +742,13 @@ partial def collectSyntaxBasedSemanticTokens (text : FileMap) : (stx : Syntax) if noHighlightKinds.contains stx.getKind then return #[] if docKinds.contains stx.getKind then - -- Docs are only highlighted in Verso format, in which case `stx[1]` is a node. + -- Verso docstrings have `stx[1]` as a syntax node; plain (CommonMark) + -- docstrings have `stx[1]` as a single atom whose source span covers + -- the docstring body. if stx[1].isAtom then - return #[] + let some ⟨startPos, endPos⟩ := stx[1].getRange? + | return #[] + return collectMarkdownTokens text startPos endPos else return collectVersoTokens text stx[1] (collectSyntaxBasedSemanticTokens text) let mut tokens := diff --git a/src/Lean/Server/ProtocolOverview.lean b/src/Lean/Server/ProtocolOverview.lean index ddd3587f8980..de9ea9d80111 100644 --- a/src/Lean/Server/ProtocolOverview.lean +++ b/src/Lean/Server/ProtocolOverview.lean @@ -249,7 +249,7 @@ def protocolOverview : Array MessageOverview := #[ kinds := #[.standard] parameterType := SemanticTokensRangeParams responseType := SemanticTokens - description := "Emitted in VS Code when a file is changed." + description := "Emitted in VS Code when a file is changed. The server returns additional semantic tokens using the standard LSP mechanism of adding them to the semantic tokens legend." }, .request { method := "textDocument/semanticTokens/full" @@ -257,7 +257,7 @@ def protocolOverview : Array MessageOverview := #[ kinds := #[.standardViolation "Instead of reporting the full semantic tokens for the full file as specified by LSP, the Lean language server will only report the semantic tokens for the part of the file that has been processed so far. If the response is incomplete, the language server periodically emits `workspace/semanticTokens/refresh` to request another `textDocument/semanticTokens/full` request from the client. This process is repeated until the file has been fully processed and all semantic tokens have been reported. We use this trick to stream semantic tokens to VS Code, despite the fact that VS Code does not support result streaming."] parameterType := SemanticTokensParams responseType := SemanticTokens - description := "Emitted in VS Code when a file is first opened, when it is changed or when VS Code receives a `workspace/semanticTokens/refresh` request from the server." + description := "Emitted in VS Code when a file is first opened, when it is changed or when VS Code receives a `workspace/semanticTokens/refresh` request from the server. The sever returns additional semantic tokens using the standard LSP mechanism of adding them to the semantic tokens legend." }, .request { method := "workspace/semanticTokens/refresh" @@ -265,7 +265,7 @@ def protocolOverview : Array MessageOverview := #[ kinds := #[.standard] parameterType := Option Empty responseType := Option Empty - description := "Emitted by the language server to request another `textDocument/semanticTokens/full` request from the client." + description := "Emitted by the language server to request another `textDocument/semanticTokens/full` request from the client. The server returns additional semantic tokens using the standard LSP mechanism of adding them to the semantic tokens legend." }, .request { method := "textDocument/inlayHint" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9b183682e486..7049f267dd55 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -305,6 +305,7 @@ add_test_pile(misc_bench *.sh BENCH PART2) add_test_pile(server *.lean) add_test_pile(server_interactive *.lean) +add_test_dir(markdown_conformance) add_test_dir(../doc/examples/compiler) add_test_dir(bench/build BENCH PART1) add_test_dir(bench/mvcgen/sym BENCH PART2) diff --git a/tests/elab/markdown_token_merge.lean b/tests/elab/markdown_token_merge.lean new file mode 100644 index 000000000000..41f7b57fc61f --- /dev/null +++ b/tests/elab/markdown_token_merge.lean @@ -0,0 +1,253 @@ +import Lean.Server.FileWorker.SemanticHighlighting + +/-! +Markup token cross-product unit tests. + +These checks parse small Markdown snippets that exercise every combination of +inline emphasis (none, bold, italic, both) with every block context (plain, +heading, blockquote, list), plus the non-composing overrides (inline code, +fenced code block, image alt). +-/ + +open Lean Lsp Server.FileWorker + +/-- Gets the token type at byte offset `byteIdx` of `input`, or `""` if no token covers it. -/ +def markupAt (input : String) (byteIdx : Nat) : String := Id.run do + let text := FileMap.ofString input + let toks := collectMarkdownTokens text 0 input.rawEndPos + let tok? := toks.findSome? fun t => do + let r ← t.stx.getRange? + guard <| r.start.byteIdx ≤ byteIdx && byteIdx < r.stop.byteIdx + SemanticTokenType.names[t.type.toNat]? + tok?.getD "" + +/-- Counts emitted tokens whose type name equals `typeName`. -/ +def tokenCount (input : String) (typeName : String) : Nat := Id.run do + let text := FileMap.ofString input + let toks := collectMarkdownTokens text 0 input.rawEndPos + let mut n := 0 + for t in toks do + if SemanticTokenType.names[t.type.toNat]? == some typeName then + n := n + 1 + return n + +/-! ## Cross-product over (emphasis, block context) + +Each line places an `x` content character inside the smallest snippet that +produces the targeted attribute combination. The expected token name follows +the `markup{Bold?}{Italic?}{Block}` convention. +-/ + +-- plain × {none, bold, italic, both} +/-- info: "markupDocText" -/ +#guard_msgs in #eval markupAt "x" 0 + +/-- info: "markupBold" -/ +#guard_msgs in #eval markupAt "**x**" 2 + +/-- info: "markupItalic" -/ +#guard_msgs in #eval markupAt "*x*" 1 + +/-- info: "markupBoldItalic" -/ +#guard_msgs in #eval markupAt "***x***" 3 + +-- heading × {none, bold, italic, both} +/-- info: "markupHeading" -/ +#guard_msgs in #eval markupAt "# x" 2 + +/-- info: "markupBoldHeading" -/ +#guard_msgs in #eval markupAt "# **x**" 4 + +/-- info: "markupItalicHeading" -/ +#guard_msgs in #eval markupAt "# *x*" 3 + +/-- info: "markupBoldItalicHeading" -/ +#guard_msgs in #eval markupAt "# ***x***" 5 + +-- blockquote × {none, bold, italic, both} +/-- info: "markupQuote" -/ +#guard_msgs in #eval markupAt "> x" 2 + +/-- info: "markupBoldQuote" -/ +#guard_msgs in #eval markupAt "> **x**" 4 + +/-- info: "markupItalicQuote" -/ +#guard_msgs in #eval markupAt "> *x*" 3 + +/-- info: "markupBoldItalicQuote" -/ +#guard_msgs in #eval markupAt "> ***x***" 5 + +-- list × {none, bold, italic, both} +/-- info: "markupList" -/ +#guard_msgs in #eval markupAt "* x" 2 + +/-- info: "markupBoldList" -/ +#guard_msgs in #eval markupAt "* **x**" 4 + +/-- info: "markupItalicList" -/ +#guard_msgs in #eval markupAt "* *x*" 3 + +/-- info: "markupBoldItalicList" -/ +#guard_msgs in #eval markupAt "* ***x***" 5 + +/-! ## Non-composing overrides + +Inline code and fenced code blocks are fixed regardless of surrounding +emphasis: their `markup*` type does not stack with the active attributes. +-/ + +/-- info: "markupInlineCode" -/ +#guard_msgs in #eval markupAt "`x`" 1 + +/-- info: "markupInlineCode" -/ +#guard_msgs in #eval markupAt "**`x`**" 3 + +/-- info: "markupCodeBlock" -/ +#guard_msgs in #eval markupAt "```\nx\n```" 4 + +-- Indented code blocks (4 leading spaces) get one `markupCodeBlock` token per +-- line, covering the whole source line including the indent. +/-- info: "markupCodeBlock" -/ +#guard_msgs in #eval markupAt " indented\n code block\n" 4 + +/-- info: 2 -/ +#guard_msgs in #eval tokenCount " indented\n code block\n" "markupCodeBlock" + +-- A code span that crosses a line break: each line of the content gets its +-- own `markupInlineCode` token (LSP semantic tokens are per-line), and the +-- delimiters remain `keyword`. +/-- info: "markupInlineCode" -/ +#guard_msgs in #eval markupAt "`a\nb`" 1 + +/-- info: "markupInlineCode" -/ +#guard_msgs in #eval markupAt "`a\nb`" 3 + +/-! ## Image alt content + +Per CommonMark §6.5, an image description has inline elements as its +contents (with the same rules as link text, plus that links are explicitly +permitted). The highlighter walks alt content recursively, so emphasis, +inline code, and links inside alt text receive their normal token types. +-/ + +-- Plain alt text gets the ambient context (here, plain). +/-- info: "markupDocText" -/ +#guard_msgs in #eval markupAt "![x](http://e.com)" 2 + +-- Bold inside alt: the inner content character is `markupBold`. +/-- info: "markupBold" -/ +#guard_msgs in #eval markupAt "![**x**](http://e.com)" 4 + +-- Inline code inside alt is still `markupInlineCode`. +/-- info: "markupInlineCode" -/ +#guard_msgs in #eval markupAt "![`x`](http://e.com)" 3 + +-- Autolink inside alt: the URL is `markupUrl`. +/-- info: "markupUrl" -/ +#guard_msgs in #eval markupAt "![](http://e.com)" 3 + +/-! ## Order independence + +The two snippets nest bold and italic in opposite orders. The inner content +character must receive the same `markupBoldItalic*` type in both +arrangements; otherwise attribute accumulation depends on walk direction. +-/ + +/-- info: "markupBoldItalic" -/ +#guard_msgs in #eval markupAt "**a *x* a**" 5 + +/-- info: "markupBoldItalic" -/ +#guard_msgs in #eval markupAt "*a **x** a*" 5 + +/-- info: "markupBoldItalicHeading" -/ +#guard_msgs in #eval markupAt "# **a *x* a**" 7 + +/-- info: "markupBoldItalicHeading" -/ +#guard_msgs in #eval markupAt "# *a **x** a*" 7 + +/-- info: "markupBoldItalicQuote" -/ +#guard_msgs in #eval markupAt "> **a *x* a**" 7 + +/-- info: "markupBoldItalicQuote" -/ +#guard_msgs in #eval markupAt "> *a **x** a*" 7 + +/-- info: "markupBoldItalicList" -/ +#guard_msgs in #eval markupAt "* **a *x* a**" 7 + +/-- info: "markupBoldItalicList" -/ +#guard_msgs in #eval markupAt "* *a **x** a*" 7 + +/-! ## Soft breaks do not produce multi-line tokens + +The inline parser emits a one-byte `.text` element covering each `\n` +between paragraph lines so downstream renderers can reproduce the line +ending. The semantic-token emitter must not turn that element into an LSP +token, since VS Code does not support multi-line semantic tokens. +-/ + +-- Soft break between two paragraph lines: byte 5 is the `\n`. +/-- info: "" -/ +#guard_msgs in #eval markupAt "hello\nworld" 5 + +-- Soft break inside a heading inline run. +/-- info: "" -/ +#guard_msgs in #eval markupAt "# a *cross\nline* b" 10 + +-- Surrounding line content still receives its expected types. +/-- info: "markupDocText" -/ +#guard_msgs in #eval markupAt "hello\nworld" 0 + +/-- info: "markupDocText" -/ +#guard_msgs in #eval markupAt "hello\nworld" 6 + +/-! ## Multi-line link reference definitions + +A ref-def label may span newlines (CommonMark §4.7) and a title may sit +on the line following the URL. Both are split at line boundaries so each +emitted token stays on a single line. +-/ + +-- Two-line label: one `markupCrossReference` token per source line. +/-- info: 2 -/ +#guard_msgs in #eval tokenCount "[foo\nbar]: https://e.com\n" "markupCrossReference" + +-- Two-line title: one `string` token per line. +/-- info: 2 -/ +#guard_msgs in #eval tokenCount "[a]: https://e.com \"title\nspans lines\"\n" "string" + +/-! ## Ref defs inside blockquotes + +`postProcessRefDefs` walks into blockquote and list children, so a ref +def inside a blockquote still produces a `markupCrossReference` token on +its label. +-/ + +/-- info: "markupCrossReference" -/ +#guard_msgs in #eval markupAt "> [foo]: https://e.com\n" 3 + +/-! ## ATX heading guards + +Seven or more `#` characters are not a valid ATX heading (CommonMark +§4.2 caps the level at six). The line falls through to a paragraph and +its content receives the plain `markupDocText` type. +-/ + +/-- info: "markupDocText" -/ +#guard_msgs in #eval markupAt "####### x" 8 + +-- Six hashes is still a heading. +/-- info: "markupHeading" -/ +#guard_msgs in #eval markupAt "###### x" 7 + +/-! ## Nested lists with mixed bullet kinds + +Different bullet characters at different nesting levels form distinct +sibling-incompatible lists. The inner list's content still receives the +`markupList` type. +-/ + +/-- info: "markupList" -/ +#guard_msgs in #eval markupAt "* outer\n - inner" 12 + +/-- info: "markupList" -/ +#guard_msgs in #eval markupAt "* outer\n + inner" 12 diff --git a/tests/markdown_conformance/README.txt b/tests/markdown_conformance/README.txt new file mode 100644 index 000000000000..944acb07b5fc --- /dev/null +++ b/tests/markdown_conformance/README.txt @@ -0,0 +1,10 @@ +This directory tests the Markdown parser used for semantic tokens against the CommonMark Spec. The +testing procedure consumes the spec as input. + +The CommonMark Spec is included verbatim, in accord with its license; it is not covered by the +Apache license of the repository as a whole. + +The specification is copyright 2024, John MacFarlane, and is distributed according to its Creative +Commons Attribution-ShareAlike 4.0 International license (CC BY-SA 4.0). + +The specification can also be found at: https://spec.commonmark.org/0.31.2/ diff --git a/tests/markdown_conformance/conformance.out.expected b/tests/markdown_conformance/conformance.out.expected new file mode 100644 index 000000000000..4d2055dfa45f --- /dev/null +++ b/tests/markdown_conformance/conformance.out.expected @@ -0,0 +1,87 @@ +Skipped: 109/655 (16%) +Overall: 546/546 (100%) + +Per section: + ATX headings: + Pass: 18/18 (100%) + Autolinks: + Pass: 17/17 (100%) + Skip: + HTML: 2 + Backslash escapes: + Pass: 12/12 (100%) + Skip: + HTML: 1 + Blank lines: + Pass: 1/1 (100%) + Block quotes: + Pass: 25/25 (100%) + Code spans: + Pass: 20/20 (100%) + Skip: + HTML: 2 + Emphasis and strong emphasis: + Pass: 123/123 (100%) + Skip: + HTML: 3 + Unicode flanking: 6 + Entity and numeric character references: + Pass: 8/8 (100%) + Skip: + HTML: 1 + Named entity: 8 + Fenced code blocks: + Pass: 29/29 (100%) + HTML blocks: + Pass: 0/0 (0%) + Skip: + HTML: 46 + Hard line breaks: + Pass: 13/13 (100%) + Skip: + HTML: 2 + Images: + Pass: 21/21 (100%) + Skip: + HTML: 1 + Indented code blocks: + Pass: 11/11 (100%) + Skip: + HTML: 1 + Link reference definitions: + Pass: 24/24 (100%) + Skip: + HTML: 2 + Unicode case-folding: 1 + Links: + Pass: 80/80 (100%) + Skip: + HTML: 8 + Unicode case-folding: 1 + Named entity: 1 + List items: + Pass: 48/48 (100%) + Lists: + Pass: 25/25 (100%) + Skip: + HTML: 2 + Paragraphs: + Pass: 8/8 (100%) + Precedence: + Pass: 1/1 (100%) + Raw HTML: + Pass: 1/1 (100%) + Skip: + HTML: 20 + Setext headings: + Pass: 26/26 (100%) + Skip: + HTML: 1 + Soft line breaks: + Pass: 2/2 (100%) + Tabs: + Pass: 11/11 (100%) + Textual content: + Pass: 3/3 (100%) + Thematic breaks: + Pass: 19/19 (100%) diff --git a/tests/markdown_conformance/run_test.sh b/tests/markdown_conformance/run_test.sh new file mode 100644 index 000000000000..dc02ac7bb319 --- /dev/null +++ b/tests/markdown_conformance/run_test.sh @@ -0,0 +1,9 @@ +# With `-v`, every diverging example is printed to stdout, so a regression +# turning a passing example into a failing one (or vice versa) shows up +# example-level in the diff against `conformance.out.expected`, not merely +# as a per-section tally shift. The runner exits 0 when every non-skipped +# example matches the spec. +capture_only "conformance" \ + lean -Dlinter.all=false --run runner.lean -v spec.txt +check_exit_is_success +check_out_file diff --git a/tests/markdown_conformance/runner.lean b/tests/markdown_conformance/runner.lean new file mode 100644 index 000000000000..e4752f4f80ab --- /dev/null +++ b/tests/markdown_conformance/runner.lean @@ -0,0 +1,776 @@ +/- +Conformance runner for the CommonMark reference test suite. + +Usage: + lean --run tests/markdown_conformance/runner.lean [path/to/spec.txt] + +Defaults to `tests/markdown_conformance/spec.txt`. + +Reads the spec source directly: examples are embedded in long-backtick +fences (`````… example`), with `.` separating the markdown input from the +expected HTML output. The literal `→` character in spec.txt represents a +tab and is converted at extraction time. Section names are tracked from +`##` headings. + +Each extracted example is fed through `parseDocument` and the +`Markdown.Html` renderer; the output is compared byte-for-byte against the +expected HTML. The runner prints a per-section pass/fail tally and exits 0 +on full agreement, 1 otherwise. + +Obtain the spec source with: + + curl -L -o tests/markdown_conformance/spec.txt \\ + https://raw.githubusercontent.com/commonmark/commonmark-spec/master/spec.txt +-/ +module +import Lean.Server.FileWorker.Markdown +import all Lean.Server.FileWorker.Markdown.Parser.Inline +import Std.Data.HashMap +import Std.Data.TreeMap.Lemmas + +open Std +open Lean.Server.FileWorker.Markdown + +/-! +## CommonMark HTML rendering (test-only) + +A simple HTML renderer for the `Block (Array Inline)` tree produced by +`Lean.Server.FileWorker.Markdown.parseDocument`. This lives inside the +conformance runner because nothing else in the toolchain needs it: the +language server emits semantic tokens, not HTML. + +The renderer aims for byte-identical output with the CommonMark reference +implementation on the constructs we support, but does not attempt to +faithfully implement the full spec. It is primarily a conformance-suite +oracle: feed Markdown to `parseDocument` plus this renderer, compare to +the expected HTML, and tally divergences. +-/ + +namespace Lean.Server.FileWorker.Markdown.Html + +private def escapeChar : Char → String + | '&' => "&" + | '<' => "<" + | '>' => ">" + | '"' => """ + | c => c.toString + +/-- HTML-escape `&`, `<`, `>`, and `"`. -/ +def escape (s : String) : String := + s.foldl (init := "") fun acc c => acc ++ escapeChar c + +/-- Substring of `source` covered by `r`. -/ +def extract (source : String) (r : Syntax.Range) : String := + String.Pos.Raw.extract source r.start r.stop + +/-- +Strips backslash escapes from a string so that `\X` becomes `X` whenever `X` +is an ASCII punctuation character; backslashes before any other character +are preserved verbatim. This matches CommonMark's normalisation of +non-inline-parsed slots like link URLs and titles, where the inline +delimiter machinery cannot run but escapes still apply. +-/ +private def unescape (s : String) : String := + let (out, pendingBackslash) := s.foldl (init := ("", false)) fun (out, escaped) c => + if escaped then + if isAsciiPunctuation c then (out.push c, false) + else (out.push '\\' |>.push c, false) + else if c == '\\' then (out, true) + else (out.push c, false) + if pendingBackslash then out.push '\\' else out + +private def hexDigit (n : Nat) : Char := + if n < 10 then Char.ofNat (n + '0'.toNat) + else Char.ofNat (n - 10 + 'A'.toNat) + +private def appendHex2 (out : String) (n : Nat) : String := + out.push '%' |>.push (hexDigit (n / 16)) |>.push (hexDigit (n % 16)) + +/-- +Whether `c` is an ASCII character that should pass through URL +percent-encoding unchanged. Matches the cmark "safe" set used by +CommonMark conformance tests: alphanumerics plus the URL-syntax +punctuation `-_.~!$&'()*+,;=:@/?#`. The `%` character is treated +specially by the caller to preserve existing percent-encoded sequences; +brackets `[`/`]` and braces are *not* in the set because cmark +percent-encodes them in autolink-style URLs. +-/ +private def isUrlSafe (c : Char) : Bool := + let n := c.toNat + ('0'.toNat ≤ n && n ≤ '9'.toNat) || + ('A'.toNat ≤ n && n ≤ 'Z'.toNat) || + ('a'.toNat ≤ n && n ≤ 'z'.toNat) || + c == '-' || c == '_' || c == '.' || c == '~' + || c == '!' || c == '$' || c == '&' || c == '\'' + || c == '(' || c == ')' || c == '*' || c == '+' + || c == ',' || c == ';' || c == '=' + || c == ':' || c == '@' || c == '/' || c == '?' + || c == '#' + +private def isHexDigit (c : Char) : Bool := + let n := c.toNat + ('0'.toNat ≤ n && n ≤ '9'.toNat) + || ('a'.toNat ≤ n && n ≤ 'f'.toNat) + || ('A'.toNat ≤ n && n ≤ 'F'.toNat) + +/-- +Appends the UTF-8 encoding of `c` to `out`, with each byte emitted as +`%HH`. Used for non-ASCII URL chars per CommonMark's conformance +expectations. +-/ +private def appendUtf8Encoded (out : String) (c : Char) : String := Id.run do + let n := c.toNat + let mut out := out + if n < 0x80 then + out := appendHex2 out n + else if n < 0x800 then + out := appendHex2 out (0xC0 + n / 0x40) + out := appendHex2 out (0x80 + n % 0x40) + else if n < 0x10000 then + out := appendHex2 out (0xE0 + n / 0x1000) + out := appendHex2 out (0x80 + (n / 0x40) % 0x40) + out := appendHex2 out (0x80 + n % 0x40) + else + out := appendHex2 out (0xF0 + n / 0x40000) + out := appendHex2 out (0x80 + (n / 0x1000) % 0x40) + out := appendHex2 out (0x80 + (n / 0x40) % 0x40) + out := appendHex2 out (0x80 + n % 0x40) + return out + +private def hexDigitVal? (c : Char) : Option Nat := + let n := c.toNat + if '0'.toNat ≤ n && n ≤ '9'.toNat then some (n - '0'.toNat) + else if 'a'.toNat ≤ n && n ≤ 'f'.toNat then some (n - 'a'.toNat + 10) + else if 'A'.toNat ≤ n && n ≤ 'F'.toNat then some (n - 'A'.toNat + 10) + else none + +private def decDigitVal? (c : Char) : Option Nat := + let n := c.toNat + if '0'.toNat ≤ n && n ≤ '9'.toNat then some (n - '0'.toNat) else none + +/-- +Decodes an HTML5 entity body — the text between `&` and `;` — to the +corresponding character. Handles numeric character references +(`#NN` decimal and `#xHH`/`#XHH` hex) and a small list of named ASCII +entities (`amp`, `lt`, `gt`, `quot`, `apos`, `nbsp`). The CommonMark +conformance suite's named-entity expectations are skipped at the +runner level, so this minimal table is enough for our purposes. +Returns `none` if the body doesn't decode to a recognised entity, in +which case the caller leaves the original `&...;` text in place. +-/ +private def decodeEntityBody (body : String) : Option String := Id.run do + let sl := body.toSlice + if sl.isEmpty then return none + if sl.startsWith '#' then + let rest := sl.drop 1 + if rest.isEmpty then return none + let isHex := rest.startsWith 'x' || rest.startsWith 'X' + let digits := if isHex then rest.drop 1 else rest + if digits.isEmpty then return none + let mut n : Nat := 0 + let mut count : Nat := 0 + for c in digits do + count := count + 1 + if count > 7 then return none + match (if isHex then hexDigitVal? c else decDigitVal? c) with + | none => return none + | some d => n := n * (if isHex then 16 else 10) + d + -- HTML5: the replacement char (U+FFFD) is used for the null + -- character, code points outside Unicode, and surrogates. + if n == 0 || n > 0x10ffff then return some (Char.ofNat 0xfffd).toString + if n ≥ 0xd800 && n ≤ 0xdfff then return some (Char.ofNat 0xfffd).toString + return some (Char.ofNat n).toString + match body with + | "amp" => return some "&" + | "lt" => return some "<" + | "gt" => return some ">" + | "quot" => return some "\"" + | "apos" => return some "'" + | "nbsp" => return some (Char.ofNat 0xa0).toString + | _ => return none + +/-- +Decodes HTML entity references in `s` — both numeric (` `/` `) +and the small named-entity table above. Unrecognised `&…;` sequences +pass through unchanged so the surrounding `escape` step renders them +verbatim. +-/ +def decodeEntities (s : String) : String := Id.run do + let mut sl := s.toSlice + let mut out := "" + while !sl.isEmpty do + if sl.startsWith '&' then + -- Search for `;` within reasonable distance, bailing on whitespace + -- or another `&`/`<` so we don't scan unbounded stretches of text. + let mut scan := sl.drop 1 + let mut body := "" + let mut count : Nat := 0 + let mut foundSemi := false + while !scan.isEmpty && count < 32 do + if scan.startsWith ';' then + foundSemi := true; break + if scan.startsWith fun c => + c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '<' || c == '&' then + break + body := body.push scan.front + scan := scan.drop 1 + count := count + 1 + if foundSemi then + match decodeEntityBody body with + | some replacement => + out := out ++ replacement + sl := scan.drop 1 + | none => + out := out.push '&' + sl := sl.drop 1 + else + out := out.push '&' + sl := sl.drop 1 + else + out := out.push sl.front + sl := sl.drop 1 + return out + +/-- +Percent-encode `s` for use as an HTML URL attribute. Characters in the +URL-safe set pass through; an existing `%XX` (with two hex digits) is +preserved verbatim so authors can include literal percent-escapes; all +other characters (including non-ASCII, which are encoded as their UTF-8 +byte sequence) become `%HH`. +-/ +def percentEncode (s : String) : String := Id.run do + let mut sl := s.toSlice + let mut out := "" + while !sl.isEmpty do + let c := sl.front + if isUrlSafe c then + out := out.push c + sl := sl.drop 1 + else if c == '%' then + let r1 := sl.drop 1 + let r2 := r1.drop 1 + match r1.front?, r2.front? with + | some d1, some d2 => + if isHexDigit d1 && isHexDigit d2 then + out := out.push '%' |>.push d1 |>.push d2 + sl := r2.drop 1 + else + out := appendUtf8Encoded out c + sl := r1 + | _, _ => + out := appendUtf8Encoded out c + sl := sl.drop 1 + else + out := appendUtf8Encoded out c + sl := sl.drop 1 + return out + +/-- +Walks `s` and for every newline drops the leading spaces or tabs that +immediately follow it. Optionally also map newlines to spaces (used for +code-span content per CommonMark §6.1; line endings inside code spans are +treated as spaces after the §4.8 paragraph normalisation). +-/ +private def normalizeContinuation (newlinesAsSpaces : Bool) (s : String) : String := + let (out, _) := s.foldl (init := ("", false)) fun (out, atLineStart) c => + if atLineStart && (c == ' ' || c == '\t') then + (out, true) + else if c == '\n' then + (out.push (if newlinesAsSpaces then ' ' else '\n'), true) + else + (out.push c, false) + out + +private def normalizeParaText (s : String) : String := + normalizeContinuation false s + +/-- +If a code span's normalised content begins *and* ends with a space and is +not made up entirely of spaces, drop one space from each end. CommonMark +§6.1 introduces this rule so that authors can write `` ` `…` ` `` to +include content beginning or ending with a literal backtick. +-/ +private def stripCodeSpanSpaces (s : String) : String := + let sl := s.toSlice + if sl.startsWith ' ' && sl.endsWith ' ' && !sl.all (· == ' ') then + (sl.drop 1 |>.dropEnd 1).toString + else + s + +/-- +Renders the verbatim contents of a code span: strips continuation-line +leading whitespace (§4.8), convert line endings to spaces (§6.1), and +remove the optional outer space pair on either side. +-/ +private def renderCodeSpan (rawSrc : String) : String := + stripCodeSpanSpaces (normalizeContinuation true rawSrc) + +/-- +Flatten an inline tree to its plain-text content (CommonMark's `alt` +attribute rule for images): drop markup, keep text and code-span text, +recurse through emphasis/link/image children. +-/ +partial def inlineToPlainText (source : String) (i : Inline) : String := + match i with + | .text r => decodeEntities (extract source r) + | .code _ content _ => extract source content + | .italic _ content _ => + content.foldl (init := "") fun acc i => acc ++ inlineToPlainText source i + | .bold _ content _ => + content.foldl (init := "") fun acc i => acc ++ inlineToPlainText source i + | .link _ text _ _ => + text.foldl (init := "") fun acc i => acc ++ inlineToPlainText source i + | .image _ _ alt _ _ => + alt.foldl (init := "") fun acc i => acc ++ inlineToPlainText source i + | .hardBreak _ => "\n" + | .autolink _ url _ _ => extract source url + +/-- Renders a single inline element. -/ +partial def renderInline (source : String) (i : Inline) : String := + match i with + | .text r => + escape (decodeEntities (normalizeParaText (extract source r))) + | .code _ content _ => + s!"{escape (renderCodeSpan (extract source content))}" + | .italic _ content _ => + let inner := content.foldl (init := "") fun acc i => acc ++ renderInline source i + s!"{inner}" + | .bold _ content _ => + let inner := content.foldl (init := "") fun acc i => acc ++ renderInline source i + s!"{inner}" + | .link _ text _ target => + let (url, title?) := match target with + | .inline _ url _ title? => + (percentEncode (decodeEntities (unescape (extract source url))), + title?.map (fun t => decodeEntities (unescape (extract source t)))) + | .reference url title? _ => + (percentEncode (decodeEntities (unescape url)), + title?.map (fun t => decodeEntities (unescape t))) + let titleAttr := match title? with + | some t => s!" title=\"{escape t}\"" + | none => "" + let inner := text.foldl (init := "") fun acc i => acc ++ renderInline source i + s!"{inner}" + | .image _ _ alt _ target => + let (url, title?) := match target with + | .inline _ url _ title? => + (percentEncode (decodeEntities (unescape (extract source url))), + title?.map (fun t => decodeEntities (unescape (extract source t)))) + | .reference url title? _ => + (percentEncode (decodeEntities (unescape url)), + title?.map (fun t => decodeEntities (unescape t))) + let titleAttr := match title? with + | some t => s!" title=\"{escape t}\"" + | none => "" + let altText := alt.foldl (init := "") fun acc i => acc ++ inlineToPlainText source i + s!"\"{escape" + | .hardBreak _ => "
\n" + | .autolink _ url _ isEmail => + -- Autolink contents are verbatim (no backslash escapes, no entity + -- references). The href is the raw URL percent-encoded; for email + -- form, prefix with `mailto:`. The visible text is the URL itself, + -- HTML-escaped only. + let urlText := extract source url + let href := if isEmail then "mailto:" ++ urlText else urlText + s!"{escape urlText}" + +private def renderInlineArray (source : String) (inlines : Array Inline) : String := + inlines.foldl (init := "") fun acc i => acc ++ renderInline source i + +/-- Renders an array of per-source-line inline arrays, joined by `\n`. -/ +private def renderInlineLines (source : String) (lines : Array (Array Inline)) : String := + Id.run do + let mut out := "" + let mut first := true + for line in lines do + if first then + first := false + else + out := out ++ "\n" + out := out ++ renderInlineArray source line + return out + + +/-- +Strips up to `n` leading ASCII spaces from a fenced-code-block content +line. Per CommonMark §4.5, when an opening fence is indented by `n` (with +`n ≤ 3`), each content line has up to `n` leading spaces removed; lines +with fewer leading spaces are stripped down to no indent. +-/ +private def stripFenceIndent (n : Nat) (line : String) : String := Id.run do + let mut sl := line.toSlice + let mut k := 0 + while k < n && sl.startsWith ' ' do + sl := sl.drop 1 + k := k + 1 + return sl.toString + +/-- Renders a single block element. -/ +partial def renderBlock (source : String) (b : Block (Array Inline)) : String := + match b with + | .paragraph lines => + s!"

{renderInlineLines source lines}

\n" + | .atxHeading hashes content _closeHashes? => + let level := hashes.stop.byteIdx - hashes.start.byteIdx + let inner := renderInlineArray source content + s!"{inner}\n" + | .setextHeading level lines _ => + s!"{renderInlineLines source lines}\n" + | .fencedCode info _ _ lines => + let langClass := match info.infoString? with + | some r => + -- Per CommonMark §4.5, only the first word of the info string is + -- emitted as the language class; subsequent words are arbitrary + -- metadata and are ignored for the `language-…` class. Backslash + -- escapes and entity references in the info string are decoded + -- (so e.g. ```` ``` foo\+bar ```` produces `class="language-foo+bar"`). + let lang := ((extract source r).takeWhile fun c => c != ' ' && c != '\t').toString + s!" class=\"language-{escape (decodeEntities (unescape lang))}\"" + | none => "" + let codeContent := lines.foldl (init := "") fun acc r => + acc ++ escape (stripFenceIndent info.openIndent (extract source r)) ++ "\n" + s!"
{codeContent}
\n" + | .indentedCode lines => + let codeContent := lines.foldl (init := "") fun acc (_, l) => + acc ++ escape l ++ "\n" + s!"
{codeContent}
\n" + | .blockquote _ children => + let inner := children.foldl (init := "") fun acc c => acc ++ renderBlock source c + s!"
\n{inner}
\n" + | .list kind tight items => + let (tag, attr) : String × String := match kind with + | .bullet _ => ("ul", "") + | .ordered start _ => + ("ol", if start == 1 then "" else s!" start=\"{start}\"") + -- A tight list-item unwraps `

` from its direct paragraph children + -- (CommonMark §5.3); a loose list-item renders each child via + -- `renderBlock`, which keeps `

` wrappers. + let renderItem (item : Block (Array Inline)) : String := + match item with + | .listItem _ children => + if !tight then + if children.isEmpty then s!"

  • \n" + else + let inner := children.foldl (init := "") fun acc c => acc ++ renderBlock source c + s!"
  • \n{inner}
    • \n" + else + let body : String := Id.run do + let mut out := "" + let mut i : Nat := 0 + while i < children.size do + let c := children[i]! + match c with + | .paragraph lines => + out := out ++ renderInlineLines source lines + if i + 1 < children.size then out := out ++ "\n" + | _ => + out := out ++ renderBlock source c + i := i + 1 + return out + let firstIsPara := children.size > 0 + && (match children[0]! with | .paragraph .. => true | _ => false) + if children.isEmpty then s!"
  • \n" + else if firstIsPara then s!"
  • {body}
    • \n" + else s!"
  • \n{body}
    • \n" + | _ => renderBlock source item + let inner := items.foldl (init := "") fun acc item => acc ++ renderItem item + s!"<{tag}{attr}>\n{inner}\n" + | .listItem _ children => + -- A `listItem` is normally rendered by its parent list (which knows the + -- tight/loose flag). The branch here handles the unusual case of a list + -- item appearing outside a list. + let inner := children.foldl (init := "") fun acc c => acc ++ renderBlock source c + s!"
  • \n{inner}
    • \n" + | .linkRefDef .. => "" + | .thematicBreak _ => "
    \n" + +/-- Renders an array of top-level blocks. -/ +def renderBlocks (source : String) (blocks : Array (Block (Array Inline))) : String := + blocks.foldl (init := "") fun acc b => acc ++ renderBlock source b + +/-- +Convenience: parse `source` as a Markdown document and render it as HTML, +suitable for direct comparison with CommonMark's reference output. +-/ +def renderToHtml (source : String) : String := + renderBlocks source (parseDocument source ⟨0⟩ source.rawEndPos) + +end Lean.Server.FileWorker.Markdown.Html + +structure SpecEntry where + markdown : String + expectedHtml : String + sectionName : String + exampleNum : Nat +deriving Inhabited + + +structure SectionStat where + pass : Nat := 0 + total : Nat := 0 + skip : HashMap String Nat := {} +deriving Inhabited + +/-- Replaces `→` (the spec's visual tab placeholder) with literal `\t`. -/ +private def detab (s : String) : String := + s.replace "→" "\t" + +/-- +Whether `line` is a long-backtick fence containing the word `example`. The +spec uses 32 backticks but we accept any length ≥ 3 for robustness. +-/ +private def isExampleOpenFence (line : String.Slice) : Bool := + line.startsWith "````````````" && line.endsWith " example" + +/-- Whether `line` is the closing fence (a run of `\`` only). -/ +private def isExampleCloseFence (line : String.Slice) : Bool := + line.startsWith "````````````" && line.all (· == '`') + +/-- Returns `some` with an explanation if the test should be skipped -/ +def skip? (md : String) : Option String := + if hasHtml md then some "HTML" + else if (md.find? "ΑΓΩ").isSome && (md.find? "αγω").isSome then some "Unicode case-folding" + else if (md.find? "ẞ").isSome && (md.find? "SS").isSome then some "Unicode case-folding" + else if hasNamedEntity md then some "Named entity" + else if hasUnicodeFlanking md then some "Unicode flanking" + else none + +where + /-- + Whether `l` contains an HTML-tag-shaped `<…>` that's *not* a + backslash-escaped `\<`. CommonMark backslash-escapes a `<` only when + the preceding char is a `\` that itself isn't escaped, so we walk + the text and skip pairs `\X` before deciding. + -/ + hasHtml (l : String.Slice) : Bool := Id.run do + let mut l := l + while !l.isEmpty do + if l.startsWith '\\' && !(l.drop 1 |>.isEmpty) then + l := l.drop 2 + else if l.startsWith '<' ∧ !(l.drop 1).isEmpty then + if l.drop 1 |>.startsWith Char.isAlpha then + -- Could be an HTML tag or an autolink. Scan to the next `>`, + -- noting whether we saw a `:` or `@` along the way and + -- whether anything autolink-disqualifying (whitespace, `<`, + -- newline) appeared. An autolink-shaped `<…>` is *not* HTML + -- — the parser handles those — so we skip past it. + let mut j := l.drop 1 + let mut hasColonOrAt := false + let mut autolinkBroken := false + while !j.isEmpty do + if j.startsWith '>' then break + if j.startsWith fun c => c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '<' then + autolinkBroken := true; break + if j.startsWith fun c => c == ':' || c == '@' then hasColonOrAt := true + j := j.drop 1 + if !autolinkBroken ∧ j.startsWith '>' ∧ hasColonOrAt then + l := j.drop 1 + else + return true + else if l.drop 1 |>.startsWith '?' then return true + else if (l.drop 1).startsWith '/' && (l.drop 2).startsWith Char.isAlpha then return true + else + if l.drop 1 |>.startsWith '!' then + let prefixes := #["!ELEMENT", "!DOCTYPE", "![CDATA", "!--"] + for pre in prefixes do + if l.drop 1 |>.startsWith pre then return true + l := l.drop 1 + else + l := l.drop 1 + return false + /-- + Whether `l` contains a named HTML entity reference (`&name;`) whose + name the renderer doesn't decode. The renderer supports + `amp`/`lt`/`gt`/`quot`/`apos`/`nbsp` plus all numeric references — + tests using only those run normally. Anything else (e.g. `ä`, + `©`) needs a full HTML5 entity table we don't carry, so we skip + those examples. + -/ + hasNamedEntity (l : String.Slice) : Bool := Id.run do + let mut i := l.startPos + while h : i ≠ l.endPos do + if h' : i.get h == '\\' ∧ i.next h ≠ l.endPos then + i := i.next (by grind) |>.next (by grind) + else if i.get h == '&' ∧ (∃ (h' : i.next h ≠ l.endPos), i.next h |>.get h' |>.isAlpha) then + let i' := i.next h + let mut j := i' + let mut sawSemi := false + while h' : j ≠ l.endPos do + let c := j.get h' + let j' := j.next h' + if c == ';' then sawSemi := true; break + if !c.isAlpha && !c.isDigit then break + j := j' + if h' : sawSemi ∧ j > i.next h then + let mut name := "" + let mut k : { x : l.Pos // x ≤ j } := ⟨i.next h, by grind⟩ + while h'' : k.val < j do + have : j ≤ l.endPos := j.le_endPos + have : k.val ≠ l.endPos := by grind + let k' : { x : l.Pos // x ≤ j } := + ⟨k.val.next (by grind), String.Slice.Pos.next_le_of_lt h''⟩ + name := name.push <| k.val.get (by grind) + k := k' + let supported := name == "amp" || name == "lt" || name == "gt" + || name == "quot" || name == "apos" || name == "nbsp" + if !supported then return true + i := i' + else + i := i.next h + return false + /-- + Whether `l` has a `*` or `_` immediately adjacent (no intervening + whitespace) to a non-ASCII character. CommonMark's emphasis flanking + rule classifies the surrounding char by Unicode general category, so + a correct verdict here would need full Pc/Pd/Pe/Pf/Pi/Po/Ps/Sc/Sk/Sm/So + data. We don't have it, so any test that relies on classifying a + non-ASCII char around an emphasis delimiter is skipped. + -/ + hasUnicodeFlanking (l : String.Slice) : Bool := Id.run do + let mut i := l.startPos + while h : i ≠ l.endPos do + let c := i.get h + let i' := i.next h + if c == '*' || c == '_' then + if (∃ (h : i ≠ l.startPos), (i.prev h |>.get String.Slice.Pos.prev_ne_endPos).toNat ≥ 0x80) then return true + if (∃ (h' : i.next h ≠ l.endPos), (i.next h |>.get h').toNat ≥ 0x80) then return true + i := i' + return false + + +/-- +Walks the spec source line by line, extracting examples and tracking the +current section. Returns the entries in source order. +-/ +private partial def extractEntries (source : String) : Array SpecEntry := Id.run do + let lines := source.split "\n" |>.toArray + let mut entries : Array SpecEntry := #[] + let mut sectionName : String := "" + let mut nextExample : Nat := 1 + let mut i : Nat := 0 + while h : i < lines.size do + let line := lines[i] + if line.startsWith "## " then + sectionName := line.drop 3 |>.copy + i := i + 1 + else if isExampleOpenFence line then + let mut mdLines : Array String := #[] + let mut htmlLines : Array String := #[] + let mut sawDot := false + let mut j := i + 1 + while hj : j < lines.size do + let l := lines[j] + if isExampleCloseFence l then + break + if l == "." && !sawDot then + sawDot := true + else if sawDot then + htmlLines := htmlLines.push l.copy + else + mdLines := mdLines.push l.copy + j := j + 1 + let markdown := detab ("\n".intercalate mdLines.toList ++ "\n") + let expectedHtml := detab ("\n".intercalate htmlLines.toList + ++ if htmlLines.isEmpty then "" else "\n") + entries := entries.push { + markdown, expectedHtml, sectionName, exampleNum := nextExample + } + nextExample := nextExample + 1 + i := j + 1 + else + i := i + 1 + return entries + +private def percent (p t : Nat) : Nat := + if t == 0 then 0 else (p * 100) / t + +structure Opts where + spec? : Option String := none + section? : Option String := none + verbose : Bool := false + showSkipped : Bool := false + +def Opts.spec (o : Opts) : String := o.spec?.getD "tests/markdown_conformance/spec.txt" + +def readOpts (opts : Opts) : List String → IO Opts + | [] => pure opts + | "--section" :: sec :: more => + readOpts { opts with «section?» := some sec } more + | "--verbose" :: more | "-v" :: more => readOpts { opts with verbose := true } more + | "--show-skipped" :: more => readOpts { opts with showSkipped := true } more + | spec :: more => + if spec.startsWith "-" then + throw <| .userError s!"Unknown option {spec}" + else if let some s := opts.spec? then + throw <| .userError s!"Can't set spec to {spec} because it's already {s}" + else + readOpts { opts with spec? := some spec } more + +public def main (args : List String) : IO UInt32 := do + let opts ← readOpts {} args + let path := opts.spec + let source ← IO.FS.readFile path + let entries := extractEntries source + if entries.isEmpty then + IO.eprintln s!"No examples extracted from {path}" + return 2 + + let mut stats : Std.HashMap String SectionStat := {} + let mut allSkipped : Std.TreeMap String (Array SpecEntry) := {} + let mut totalPass := 0 + let mut totalCount := 0 + let mut totalSkip := 0 + for entry in entries do + if let some s := opts.section? then + if entry.sectionName ≠ s then continue + if let some reason := skip? entry.markdown then + stats := stats.alter entry.sectionName fun s? => + let s := s?.getD {} + some { s with skip := s.skip.alter reason (some <| ·.getD 0 + 1) } + allSkipped := allSkipped.alter reason fun xs? => + xs?.getD #[] |>.push entry + totalSkip := totalSkip + 1 + else + let actual := Html.renderToHtml entry.markdown + let passed := actual == entry.expectedHtml + stats := stats.alter entry.sectionName fun s? => + let s := s?.getD {} + some { s with pass := s.pass + (if passed then 1 else 0), total := s.total + 1 } + if passed then totalPass := totalPass + 1 + if opts.verbose && !passed then + IO.println "FAILED:" + IO.println entry.markdown + IO.println "EXPECTED:" + IO.println entry.expectedHtml + IO.println "ACTUAL:" + IO.println actual + totalCount := totalCount + 1 + + IO.println s!"Skipped: {totalSkip}/{totalCount} ({percent totalSkip totalCount}%)" + IO.println s!"Overall: {totalPass}/{totalCount - totalSkip} ({percent totalPass (totalCount - totalSkip)}%)" + IO.println "" + IO.println "Per section:" + let sortedSections := stats.toList.mergeSort (fun a b => a.1 < b.1) + for (name, stat) in sortedSections do + IO.println s!" {name}:" + IO.println s!" Pass: {stat.pass}/{stat.total} ({percent stat.pass stat.total}%)" + unless stat.skip.isEmpty do + IO.println s!" Skip:" + for (reason, count) in stat.skip do + IO.println s!" {reason}: {count}" + + if opts.showSkipped then + IO.println s!"Skipped:" + for h : reason in allSkipped.keys do + have : reason ∈ allSkipped := by grind + IO.println s!"-- {reason} --" + for entry in allSkipped[reason] do + let actual := Html.renderToHtml entry.markdown + IO.println s!"SKIPPED ({reason}):" + IO.println entry.markdown + IO.println "EXPECTED:" + IO.println entry.expectedHtml + IO.println "ACTUAL:" + IO.println actual + + return if totalPass + totalSkip == totalCount then 0 else 1 diff --git a/tests/markdown_conformance/spec.txt b/tests/markdown_conformance/spec.txt new file mode 100644 index 000000000000..bf752d89f878 --- /dev/null +++ b/tests/markdown_conformance/spec.txt @@ -0,0 +1,9811 @@ +--- +title: CommonMark Spec +author: John MacFarlane +version: '0.31.2' +date: '2024-01-28' +license: '[CC-BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)' +... + +# Introduction + +## What is Markdown? + +Markdown is a plain text format for writing structured documents, +based on conventions for indicating formatting in email +and usenet posts. It was developed by John Gruber (with +help from Aaron Swartz) and released in 2004 in the form of a +[syntax description](https://daringfireball.net/projects/markdown/syntax) +and a Perl script (`Markdown.pl`) for converting Markdown to +HTML. In the next decade, dozens of implementations were +developed in many languages. Some extended the original +Markdown syntax with conventions for footnotes, tables, and +other document elements. Some allowed Markdown documents to be +rendered in formats other than HTML. Websites like Reddit, +StackOverflow, and GitHub had millions of people using Markdown. +And Markdown started to be used beyond the web, to author books, +articles, slide shows, letters, and lecture notes. + +What distinguishes Markdown from many other lightweight markup +syntaxes, which are often easier to write, is its readability. +As Gruber writes: + +> The overriding design goal for Markdown's formatting syntax is +> to make it as readable as possible. The idea is that a +> Markdown-formatted document should be publishable as-is, as +> plain text, without looking like it's been marked up with tags +> or formatting instructions. +> () + +The point can be illustrated by comparing a sample of +[AsciiDoc](https://asciidoc.org/) with +an equivalent sample of Markdown. Here is a sample of +AsciiDoc from the AsciiDoc manual: + +``` +1. List item one. ++ +List item one continued with a second paragraph followed by an +Indented block. ++ +................. +$ ls *.sh +$ mv *.sh ~/tmp +................. ++ +List item continued with a third paragraph. + +2. List item two continued with an open block. ++ +-- +This paragraph is part of the preceding list item. + +a. This list is nested and does not require explicit item +continuation. ++ +This paragraph is part of the preceding list item. + +b. List item b. + +This paragraph belongs to item two of the outer list. +-- +``` + +And here is the equivalent in Markdown: +``` +1. List item one. + + List item one continued with a second paragraph followed by an + Indented block. + + $ ls *.sh + $ mv *.sh ~/tmp + + List item continued with a third paragraph. + +2. List item two continued with an open block. + + This paragraph is part of the preceding list item. + + 1. This list is nested and does not require explicit item continuation. + + This paragraph is part of the preceding list item. + + 2. List item b. + + This paragraph belongs to item two of the outer list. +``` + +The AsciiDoc version is, arguably, easier to write. You don't need +to worry about indentation. But the Markdown version is much easier +to read. The nesting of list items is apparent to the eye in the +source, not just in the processed document. + +## Why is a spec needed? + +John Gruber's [canonical description of Markdown's +syntax](https://daringfireball.net/projects/markdown/syntax) +does not specify the syntax unambiguously. Here are some examples of +questions it does not answer: + +1. How much indentation is needed for a sublist? The spec says that + continuation paragraphs need to be indented four spaces, but is + not fully explicit about sublists. It is natural to think that + they, too, must be indented four spaces, but `Markdown.pl` does + not require that. This is hardly a "corner case," and divergences + between implementations on this issue often lead to surprises for + users in real documents. (See [this comment by John + Gruber](https://web.archive.org/web/20170611172104/http://article.gmane.org/gmane.text.markdown.general/1997).) + +2. Is a blank line needed before a block quote or heading? + Most implementations do not require the blank line. However, + this can lead to unexpected results in hard-wrapped text, and + also to ambiguities in parsing (note that some implementations + put the heading inside the blockquote, while others do not). + (John Gruber has also spoken [in favor of requiring the blank + lines](https://web.archive.org/web/20170611172104/http://article.gmane.org/gmane.text.markdown.general/2146).) + +3. Is a blank line needed before an indented code block? + (`Markdown.pl` requires it, but this is not mentioned in the + documentation, and some implementations do not require it.) + + ``` markdown + paragraph + code? + ``` + +4. What is the exact rule for determining when list items get + wrapped in `

    ` tags? Can a list be partially "loose" and partially + "tight"? What should we do with a list like this? + + ``` markdown + 1. one + + 2. two + 3. three + ``` + + Or this? + + ``` markdown + 1. one + - a + + - b + 2. two + ``` + + (There are some relevant comments by John Gruber + [here](https://web.archive.org/web/20170611172104/http://article.gmane.org/gmane.text.markdown.general/2554).) + +5. Can list markers be indented? Can ordered list markers be right-aligned? + + ``` markdown + 8. item 1 + 9. item 2 + 10. item 2a + ``` + +6. Is this one list with a thematic break in its second item, + or two lists separated by a thematic break? + + ``` markdown + * a + * * * * * + * b + ``` + +7. When list markers change from numbers to bullets, do we have + two lists or one? (The Markdown syntax description suggests two, + but the perl scripts and many other implementations produce one.) + + ``` markdown + 1. fee + 2. fie + - foe + - fum + ``` + +8. What are the precedence rules for the markers of inline structure? + For example, is the following a valid link, or does the code span + take precedence ? + + ``` markdown + [a backtick (`)](/url) and [another backtick (`)](/url). + ``` + +9. What are the precedence rules for markers of emphasis and strong + emphasis? For example, how should the following be parsed? + + ``` markdown + *foo *bar* baz* + ``` + +10. What are the precedence rules between block-level and inline-level + structure? For example, how should the following be parsed? + + ``` markdown + - `a long code span can contain a hyphen like this + - and it can screw things up` + ``` + +11. Can list items include section headings? (`Markdown.pl` does not + allow this, but does allow blockquotes to include headings.) + + ``` markdown + - # Heading + ``` + +12. Can list items be empty? + + ``` markdown + * a + * + * b + ``` + +13. Can link references be defined inside block quotes or list items? + + ``` markdown + > Blockquote [foo]. + > + > [foo]: /url + ``` + +14. If there are multiple definitions for the same reference, which takes + precedence? + + ``` markdown + [foo]: /url1 + [foo]: /url2 + + [foo][] + ``` + +In the absence of a spec, early implementers consulted `Markdown.pl` +to resolve these ambiguities. But `Markdown.pl` was quite buggy, and +gave manifestly bad results in many cases, so it was not a +satisfactory replacement for a spec. + +Because there is no unambiguous spec, implementations have diverged +considerably. As a result, users are often surprised to find that +a document that renders one way on one system (say, a GitHub wiki) +renders differently on another (say, converting to docbook using +pandoc). To make matters worse, because nothing in Markdown counts +as a "syntax error," the divergence often isn't discovered right away. + +## About this document + +This document attempts to specify Markdown syntax unambiguously. +It contains many examples with side-by-side Markdown and +HTML. These are intended to double as conformance tests. An +accompanying script `spec_tests.py` can be used to run the tests +against any Markdown program: + + python test/spec_tests.py --spec spec.txt --program PROGRAM + +Since this document describes how Markdown is to be parsed into +an abstract syntax tree, it would have made sense to use an abstract +representation of the syntax tree instead of HTML. But HTML is capable +of representing the structural distinctions we need to make, and the +choice of HTML for the tests makes it possible to run the tests against +an implementation without writing an abstract syntax tree renderer. + +Note that not every feature of the HTML samples is mandated by +the spec. For example, the spec says what counts as a link +destination, but it doesn't mandate that non-ASCII characters in +the URL be percent-encoded. To use the automatic tests, +implementers will need to provide a renderer that conforms to +the expectations of the spec examples (percent-encoding +non-ASCII characters in URLs). But a conforming implementation +can use a different renderer and may choose not to +percent-encode non-ASCII characters in URLs. + +This document is generated from a text file, `spec.txt`, written +in Markdown with a small extension for the side-by-side tests. +The script `tools/makespec.py` can be used to convert `spec.txt` into +HTML or CommonMark (which can then be converted into other formats). + +In the examples, the `→` character is used to represent tabs. + +# Preliminaries + +## Characters and lines + +Any sequence of [characters] is a valid CommonMark +document. + +A [character](@) is a Unicode code point. Although some +code points (for example, combining accents) do not correspond to +characters in an intuitive sense, all code points count as characters +for purposes of this spec. + +This spec does not specify an encoding; it thinks of lines as composed +of [characters] rather than bytes. A conforming parser may be limited +to a certain encoding. + +A [line](@) is a sequence of zero or more [characters] +other than line feed (`U+000A`) or carriage return (`U+000D`), +followed by a [line ending] or by the end of file. + +A [line ending](@) is a line feed (`U+000A`), a carriage return +(`U+000D`) not followed by a line feed, or a carriage return and a +following line feed. + +A line containing no characters, or a line containing only spaces +(`U+0020`) or tabs (`U+0009`), is called a [blank line](@). + +The following definitions of character classes will be used in this spec: + +A [Unicode whitespace character](@) is a character in the Unicode `Zs` general +category, or a tab (`U+0009`), line feed (`U+000A`), form feed (`U+000C`), or +carriage return (`U+000D`). + +[Unicode whitespace](@) is a sequence of one or more +[Unicode whitespace characters]. + +A [tab](@) is `U+0009`. + +A [space](@) is `U+0020`. + +An [ASCII control character](@) is a character between `U+0000–1F` (both +including) or `U+007F`. + +An [ASCII punctuation character](@) +is `!`, `"`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, +`*`, `+`, `,`, `-`, `.`, `/` (U+0021–2F), +`:`, `;`, `<`, `=`, `>`, `?`, `@` (U+003A–0040), +`[`, `\`, `]`, `^`, `_`, `` ` `` (U+005B–0060), +`{`, `|`, `}`, or `~` (U+007B–007E). + +A [Unicode punctuation character](@) is a character in the Unicode `P` +(punctuation) or `S` (symbol) general categories. + +## Tabs + +Tabs in lines are not expanded to [spaces]. However, +in contexts where spaces help to define block structure, +tabs behave as if they were replaced by spaces with a tab stop +of 4 characters. + +Thus, for example, a tab can be used instead of four spaces +in an indented code block. (Note, however, that internal +tabs are passed through as literal tabs, not expanded to +spaces.) + +```````````````````````````````` example +→foo→baz→→bim +. +

    foo→baz→→bim
+
    +```````````````````````````````` + +```````````````````````````````` example + →foo→baz→→bim +. +
    foo→baz→→bim
+
    +```````````````````````````````` + +```````````````````````````````` example + a→a + ὐ→a +. +
    a→a
+ὐ→a
+
    +```````````````````````````````` + +In the following example, a continuation paragraph of a list +item is indented with a tab; this has exactly the same effect +as indentation with four spaces would: + +```````````````````````````````` example + - foo + +→bar +. + +```````````````````````````````` + +```````````````````````````````` example +- foo + +→→bar +. + +```````````````````````````````` + +Normally the `>` that begins a block quote may be followed +optionally by a space, which is not considered part of the +content. In the following case `>` is followed by a tab, +which is treated as if it were expanded into three spaces. +Since one of these spaces is considered part of the +delimiter, `foo` is considered to be indented six spaces +inside the block quote context, so we get an indented +code block starting with two spaces. + +```````````````````````````````` example +>→→foo +. +
    +
      foo
+
    +
    +```````````````````````````````` + +```````````````````````````````` example +-→→foo +. + +```````````````````````````````` + + +```````````````````````````````` example + foo +→bar +. +
    foo
+bar
+
    +```````````````````````````````` + +```````````````````````````````` example + - foo + - bar +→ - baz +. + +```````````````````````````````` + +```````````````````````````````` example +#→Foo +. +

    Foo

    +```````````````````````````````` + +```````````````````````````````` example +*→*→*→ +. +
    +```````````````````````````````` + + +## Insecure characters + +For security reasons, the Unicode character `U+0000` must be replaced +with the REPLACEMENT CHARACTER (`U+FFFD`). + + +## Backslash escapes + +Any ASCII punctuation character may be backslash-escaped: + +```````````````````````````````` example +\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~ +. +

    !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~

    +```````````````````````````````` + + +Backslashes before other characters are treated as literal +backslashes: + +```````````````````````````````` example +\→\A\a\ \3\φ\« +. +

    \→\A\a\ \3\φ\«

    +```````````````````````````````` + + +Escaped characters are treated as regular characters and do +not have their usual Markdown meanings: + +```````````````````````````````` example +\*not emphasized* +\
    not a tag +\[not a link](/foo) +\`not code` +1\. not a list +\* not a list +\# not a heading +\[foo]: /url "not a reference" +\ö not a character entity +. +

    *not emphasized* +<br/> not a tag +[not a link](/foo) +`not code` +1. not a list +* not a list +# not a heading +[foo]: /url "not a reference" +&ouml; not a character entity

    +```````````````````````````````` + + +If a backslash is itself escaped, the following character is not: + +```````````````````````````````` example +\\*emphasis* +. +

    \emphasis

    +```````````````````````````````` + + +A backslash at the end of the line is a [hard line break]: + +```````````````````````````````` example +foo\ +bar +. +

    foo
    +bar

    +```````````````````````````````` + + +Backslash escapes do not work in code blocks, code spans, autolinks, or +raw HTML: + +```````````````````````````````` example +`` \[\` `` +. +

    \[\`

    +```````````````````````````````` + + +```````````````````````````````` example + \[\] +. +
    \[\]
+
    +```````````````````````````````` + + +```````````````````````````````` example +~~~ +\[\] +~~~ +. +
    \[\]
+
    +```````````````````````````````` + + +```````````````````````````````` example + +. +

    https://example.com?find=\*

    +```````````````````````````````` + + +```````````````````````````````` example + +. + +```````````````````````````````` + + +But they work in all other contexts, including URLs and link titles, +link references, and [info strings] in [fenced code blocks]: + +```````````````````````````````` example +[foo](/bar\* "ti\*tle") +. +

    foo

    +```````````````````````````````` + + +```````````````````````````````` example +[foo] + +[foo]: /bar\* "ti\*tle" +. +

    foo

    +```````````````````````````````` + + +```````````````````````````````` example +``` foo\+bar +foo +``` +. +
    foo
+
    +```````````````````````````````` + + +## Entity and numeric character references + +Valid HTML entity references and numeric character references +can be used in place of the corresponding Unicode character, +with the following exceptions: + +- Entity and character references are not recognized in code + blocks and code spans. + +- Entity and character references cannot stand in place of + special characters that define structural elements in + CommonMark. For example, although `*` can be used + in place of a literal `*` character, `*` cannot replace + `*` in emphasis delimiters, bullet list markers, or thematic + breaks. + +Conforming CommonMark parsers need not store information about +whether a particular character was represented in the source +using a Unicode character or an entity reference. + +[Entity references](@) consist of `&` + any of the valid +HTML5 entity names + `;`. The +document +is used as an authoritative source for the valid entity +references and their corresponding code points. + +```````````````````````````````` example +  & © Æ Ď +¾ ℋ ⅆ +∲ ≧̸ +. +

      & © Æ Ď +¾ ℋ ⅆ +∲ ≧̸

    +```````````````````````````````` + + +[Decimal numeric character +references](@) +consist of `&#` + a string of 1--7 arabic digits + `;`. A +numeric character reference is parsed as the corresponding +Unicode character. Invalid Unicode code points will be replaced by +the REPLACEMENT CHARACTER (`U+FFFD`). For security reasons, +the code point `U+0000` will also be replaced by `U+FFFD`. + +```````````````````````````````` example +# Ӓ Ϡ � +. +

    # Ӓ Ϡ �

    +```````````````````````````````` + + +[Hexadecimal numeric character +references](@) consist of `&#` + +either `X` or `x` + a string of 1-6 hexadecimal digits + `;`. +They too are parsed as the corresponding Unicode character (this +time specified with a hexadecimal numeral instead of decimal). + +```````````````````````````````` example +" ആ ಫ +. +

    " ആ ಫ

    +```````````````````````````````` + + +Here are some nonentities: + +```````````````````````````````` example +  &x; &#; &#x; +� +&#abcdef0; +&ThisIsNotDefined; &hi?; +. +

    &nbsp &x; &#; &#x; +&#87654321; +&#abcdef0; +&ThisIsNotDefined; &hi?;

    +```````````````````````````````` + + +Although HTML5 does accept some entity references +without a trailing semicolon (such as `©`), these are not +recognized here, because it makes the grammar too ambiguous: + +```````````````````````````````` example +© +. +

    &copy

    +```````````````````````````````` + + +Strings that are not on the list of HTML5 named entities are not +recognized as entity references either: + +```````````````````````````````` example +&MadeUpEntity; +. +

    &MadeUpEntity;

    +```````````````````````````````` + + +Entity and numeric character references are recognized in any +context besides code spans or code blocks, including +URLs, [link titles], and [fenced code block][] [info strings]: + +```````````````````````````````` example + +. + +```````````````````````````````` + + +```````````````````````````````` example +[foo](/föö "föö") +. +

    foo

    +```````````````````````````````` + + +```````````````````````````````` example +[foo] + +[foo]: /föö "föö" +. +

    foo

    +```````````````````````````````` + + +```````````````````````````````` example +``` föö +foo +``` +. +
    foo
+
    +```````````````````````````````` + + +Entity and numeric character references are treated as literal +text in code spans and code blocks: + +```````````````````````````````` example +`föö` +. +

    f&ouml;&ouml;

    +```````````````````````````````` + + +```````````````````````````````` example + föfö +. +
    f&ouml;f&ouml;
+
    +```````````````````````````````` + + +Entity and numeric character references cannot be used +in place of symbols indicating structure in CommonMark +documents. + +```````````````````````````````` example +*foo* +*foo* +. +

    *foo* +foo

    +```````````````````````````````` + +```````````````````````````````` example +* foo + +* foo +. +

    * foo

    +
      +
    • foo
      • +
    +```````````````````````````````` + +```````````````````````````````` example +foo bar +. +

    foo + +bar

    +```````````````````````````````` + +```````````````````````````````` example + foo +. +

    →foo

    +```````````````````````````````` + + +```````````````````````````````` example +[a](url "tit") +. +

    [a](url "tit")

    +```````````````````````````````` + + + +# Blocks and inlines + +We can think of a document as a sequence of +[blocks](@)---structural elements like paragraphs, block +quotations, lists, headings, rules, and code blocks. Some blocks (like +block quotes and list items) contain other blocks; others (like +headings and paragraphs) contain [inline](@) content---text, +links, emphasized text, images, code spans, and so on. + +## Precedence + +Indicators of block structure always take precedence over indicators +of inline structure. So, for example, the following is a list with +two items, not a list with one item containing a code span: + +```````````````````````````````` example +- `one +- two` +. +
      +
    • `one
      • +
    • two`
      • +
    +```````````````````````````````` + + +This means that parsing can proceed in two steps: first, the block +structure of the document can be discerned; second, text lines inside +paragraphs, headings, and other block constructs can be parsed for inline +structure. The second step requires information about link reference +definitions that will be available only at the end of the first +step. Note that the first step requires processing lines in sequence, +but the second can be parallelized, since the inline parsing of +one block element does not affect the inline parsing of any other. + +## Container blocks and leaf blocks + +We can divide blocks into two types: +[container blocks](#container-blocks), +which can contain other blocks, and [leaf blocks](#leaf-blocks), +which cannot. + +# Leaf blocks + +This section describes the different kinds of leaf block that make up a +Markdown document. + +## Thematic breaks + +A line consisting of optionally up to three spaces of indentation, followed by a +sequence of three or more matching `-`, `_`, or `*` characters, each followed +optionally by any number of spaces or tabs, forms a +[thematic break](@). + +```````````````````````````````` example +*** +--- +___ +. +
    +
    +
    +```````````````````````````````` + + +Wrong characters: + +```````````````````````````````` example ++++ +. +

    +++

    +```````````````````````````````` + + +```````````````````````````````` example +=== +. +

    ===

    +```````````````````````````````` + + +Not enough characters: + +```````````````````````````````` example +-- +** +__ +. +

    -- +** +__

    +```````````````````````````````` + + +Up to three spaces of indentation are allowed: + +```````````````````````````````` example + *** + *** + *** +. +
    +
    +
    +```````````````````````````````` + + +Four spaces of indentation is too many: + +```````````````````````````````` example + *** +. +
    ***
+
    +```````````````````````````````` + + +```````````````````````````````` example +Foo + *** +. +

    Foo +***

    +```````````````````````````````` + + +More than three characters may be used: + +```````````````````````````````` example +_____________________________________ +. +
    +```````````````````````````````` + + +Spaces and tabs are allowed between the characters: + +```````````````````````````````` example + - - - +. +
    +```````````````````````````````` + + +```````````````````````````````` example + ** * ** * ** * ** +. +
    +```````````````````````````````` + + +```````````````````````````````` example +- - - - +. +
    +```````````````````````````````` + + +Spaces and tabs are allowed at the end: + +```````````````````````````````` example +- - - - +. +
    +```````````````````````````````` + + +However, no other characters may occur in the line: + +```````````````````````````````` example +_ _ _ _ a + +a------ + +---a--- +. +

    _ _ _ _ a

    +

    a------

    +

    ---a---

    +```````````````````````````````` + + +It is required that all of the characters other than spaces or tabs be the same. +So, this is not a thematic break: + +```````````````````````````````` example + *-* +. +

    -

    +```````````````````````````````` + + +Thematic breaks do not need blank lines before or after: + +```````````````````````````````` example +- foo +*** +- bar +. +
      +
    • foo
      • +
    +
    +
      +
    • bar
      • +
    +```````````````````````````````` + + +Thematic breaks can interrupt a paragraph: + +```````````````````````````````` example +Foo +*** +bar +. +

    Foo

    +
    +

    bar

    +```````````````````````````````` + + +If a line of dashes that meets the above conditions for being a +thematic break could also be interpreted as the underline of a [setext +heading], the interpretation as a +[setext heading] takes precedence. Thus, for example, +this is a setext heading, not a paragraph followed by a thematic break: + +```````````````````````````````` example +Foo +--- +bar +. +

    Foo

    +

    bar

    +```````````````````````````````` + + +When both a thematic break and a list item are possible +interpretations of a line, the thematic break takes precedence: + +```````````````````````````````` example +* Foo +* * * +* Bar +. +
      +
    • Foo
      • +
    +
    +
      +
    • Bar
      • +
    +```````````````````````````````` + + +If you want a thematic break in a list item, use a different bullet: + +```````````````````````````````` example +- Foo +- * * * +. +
      +
    • Foo
      • +
    • +
      +
      • +
    +```````````````````````````````` + + +## ATX headings + +An [ATX heading](@) +consists of a string of characters, parsed as inline content, between an +opening sequence of 1--6 unescaped `#` characters and an optional +closing sequence of any number of unescaped `#` characters. +The opening sequence of `#` characters must be followed by spaces or tabs, or +by the end of line. The optional closing sequence of `#`s must be preceded by +spaces or tabs and may be followed by spaces or tabs only. The opening +`#` character may be preceded by up to three spaces of indentation. The raw +contents of the heading are stripped of leading and trailing space or tabs +before being parsed as inline content. The heading level is equal to the number +of `#` characters in the opening sequence. + +Simple headings: + +```````````````````````````````` example +# foo +## foo +### foo +#### foo +##### foo +###### foo +. +

    foo

    +

    foo

    +

    foo

    +

    foo

    +
    foo
    +
    foo
    +```````````````````````````````` + + +More than six `#` characters is not a heading: + +```````````````````````````````` example +####### foo +. +

    ####### foo

    +```````````````````````````````` + + +At least one space or tab is required between the `#` characters and the +heading's contents, unless the heading is empty. Note that many +implementations currently do not require the space. However, the +space was required by the +[original ATX implementation](http://www.aaronsw.com/2002/atx/atx.py), +and it helps prevent things like the following from being parsed as +headings: + +```````````````````````````````` example +#5 bolt + +#hashtag +. +

    #5 bolt

    +

    #hashtag

    +```````````````````````````````` + + +This is not a heading, because the first `#` is escaped: + +```````````````````````````````` example +\## foo +. +

    ## foo

    +```````````````````````````````` + + +Contents are parsed as inlines: + +```````````````````````````````` example +# foo *bar* \*baz\* +. +

    foo bar *baz*

    +```````````````````````````````` + + +Leading and trailing spaces or tabs are ignored in parsing inline content: + +```````````````````````````````` example +# foo +. +

    foo

    +```````````````````````````````` + + +Up to three spaces of indentation are allowed: + +```````````````````````````````` example + ### foo + ## foo + # foo +. +

    foo

    +

    foo

    +

    foo

    +```````````````````````````````` + + +Four spaces of indentation is too many: + +```````````````````````````````` example + # foo +. +
    # foo
+
    +```````````````````````````````` + + +```````````````````````````````` example +foo + # bar +. +

    foo +# bar

    +```````````````````````````````` + + +A closing sequence of `#` characters is optional: + +```````````````````````````````` example +## foo ## + ### bar ### +. +

    foo

    +

    bar

    +```````````````````````````````` + + +It need not be the same length as the opening sequence: + +```````````````````````````````` example +# foo ################################## +##### foo ## +. +

    foo

    +
    foo
    +```````````````````````````````` + + +Spaces or tabs are allowed after the closing sequence: + +```````````````````````````````` example +### foo ### +. +

    foo

    +```````````````````````````````` + + +A sequence of `#` characters with anything but spaces or tabs following it +is not a closing sequence, but counts as part of the contents of the +heading: + +```````````````````````````````` example +### foo ### b +. +

    foo ### b

    +```````````````````````````````` + + +The closing sequence must be preceded by a space or tab: + +```````````````````````````````` example +# foo# +. +

    foo#

    +```````````````````````````````` + + +Backslash-escaped `#` characters do not count as part +of the closing sequence: + +```````````````````````````````` example +### foo \### +## foo #\## +# foo \# +. +

    foo ###

    +

    foo ###

    +

    foo #

    +```````````````````````````````` + + +ATX headings need not be separated from surrounding content by blank +lines, and they can interrupt paragraphs: + +```````````````````````````````` example +**** +## foo +**** +. +
    +

    foo

    +
    +```````````````````````````````` + + +```````````````````````````````` example +Foo bar +# baz +Bar foo +. +

    Foo bar

    +

    baz

    +

    Bar foo

    +```````````````````````````````` + + +ATX headings can be empty: + +```````````````````````````````` example +## +# +### ### +. +

    +

    +

    +```````````````````````````````` + + +## Setext headings + +A [setext heading](@) consists of one or more +lines of text, not interrupted by a blank line, of which the first line does not +have more than 3 spaces of indentation, followed by +a [setext heading underline]. The lines of text must be such +that, were they not followed by the setext heading underline, +they would be interpreted as a paragraph: they cannot be +interpretable as a [code fence], [ATX heading][ATX headings], +[block quote][block quotes], [thematic break][thematic breaks], +[list item][list items], or [HTML block][HTML blocks]. + +A [setext heading underline](@) is a sequence of +`=` characters or a sequence of `-` characters, with no more than 3 +spaces of indentation and any number of trailing spaces or tabs. + +The heading is a level 1 heading if `=` characters are used in +the [setext heading underline], and a level 2 heading if `-` +characters are used. The contents of the heading are the result +of parsing the preceding lines of text as CommonMark inline +content. + +In general, a setext heading need not be preceded or followed by a +blank line. However, it cannot interrupt a paragraph, so when a +setext heading comes after a paragraph, a blank line is needed between +them. + +Simple examples: + +```````````````````````````````` example +Foo *bar* +========= + +Foo *bar* +--------- +. +

    Foo bar

    +

    Foo bar

    +```````````````````````````````` + + +The content of the header may span more than one line: + +```````````````````````````````` example +Foo *bar +baz* +==== +. +

    Foo bar +baz

    +```````````````````````````````` + +The contents are the result of parsing the headings's raw +content as inlines. The heading's raw content is formed by +concatenating the lines and removing initial and final +spaces or tabs. + +```````````````````````````````` example + Foo *bar +baz*→ +==== +. +

    Foo bar +baz

    +```````````````````````````````` + + +The underlining can be any length: + +```````````````````````````````` example +Foo +------------------------- + +Foo += +. +

    Foo

    +

    Foo

    +```````````````````````````````` + + +The heading content can be preceded by up to three spaces of indentation, and +need not line up with the underlining: + +```````````````````````````````` example + Foo +--- + + Foo +----- + + Foo + === +. +

    Foo

    +

    Foo

    +

    Foo

    +```````````````````````````````` + + +Four spaces of indentation is too many: + +```````````````````````````````` example + Foo + --- + + Foo +--- +. +
    Foo
+---
+
+Foo
+
    +
    +```````````````````````````````` + + +The setext heading underline can be preceded by up to three spaces of +indentation, and may have trailing spaces or tabs: + +```````````````````````````````` example +Foo + ---- +. +

    Foo

    +```````````````````````````````` + + +Four spaces of indentation is too many: + +```````````````````````````````` example +Foo + --- +. +

    Foo +---

    +```````````````````````````````` + + +The setext heading underline cannot contain internal spaces or tabs: + +```````````````````````````````` example +Foo += = + +Foo +--- - +. +

    Foo += =

    +

    Foo

    +
    +```````````````````````````````` + + +Trailing spaces or tabs in the content line do not cause a hard line break: + +```````````````````````````````` example +Foo +----- +. +

    Foo

    +```````````````````````````````` + + +Nor does a backslash at the end: + +```````````````````````````````` example +Foo\ +---- +. +

    Foo\

    +```````````````````````````````` + + +Since indicators of block structure take precedence over +indicators of inline structure, the following are setext headings: + +```````````````````````````````` example +`Foo +---- +` + + +. +

    `Foo

    +

    `

    +

    <a title="a lot

    +

    of dashes"/>

    +```````````````````````````````` + + +The setext heading underline cannot be a [lazy continuation +line] in a list item or block quote: + +```````````````````````````````` example +> Foo +--- +. +
    +

    Foo

    +
    +
    +```````````````````````````````` + + +```````````````````````````````` example +> foo +bar +=== +. +
    +

    foo +bar +===

    +
    +```````````````````````````````` + + +```````````````````````````````` example +- Foo +--- +. +
      +
    • Foo
      • +
    +
    +```````````````````````````````` + + +A blank line is needed between a paragraph and a following +setext heading, since otherwise the paragraph becomes part +of the heading's content: + +```````````````````````````````` example +Foo +Bar +--- +. +

    Foo +Bar

    +```````````````````````````````` + + +But in general a blank line is not required before or after +setext headings: + +```````````````````````````````` example +--- +Foo +--- +Bar +--- +Baz +. +
    +

    Foo

    +

    Bar

    +

    Baz

    +```````````````````````````````` + + +Setext headings cannot be empty: + +```````````````````````````````` example + +==== +. +

    ====

    +```````````````````````````````` + + +Setext heading text lines must not be interpretable as block +constructs other than paragraphs. So, the line of dashes +in these examples gets interpreted as a thematic break: + +```````````````````````````````` example +--- +--- +. +
    +
    +```````````````````````````````` + + +```````````````````````````````` example +- foo +----- +. +
      +
    • foo
      • +
    +
    +```````````````````````````````` + + +```````````````````````````````` example + foo +--- +. +
    foo
+
    +
    +```````````````````````````````` + + +```````````````````````````````` example +> foo +----- +. +
    +

    foo

    +
    +
    +```````````````````````````````` + + +If you want a heading with `> foo` as its literal text, you can +use backslash escapes: + +```````````````````````````````` example +\> foo +------ +. +

    > foo

    +```````````````````````````````` + + +**Compatibility note:** Most existing Markdown implementations +do not allow the text of setext headings to span multiple lines. +But there is no consensus about how to interpret + +``` markdown +Foo +bar +--- +baz +``` + +One can find four different interpretations: + +1. paragraph "Foo", heading "bar", paragraph "baz" +2. paragraph "Foo bar", thematic break, paragraph "baz" +3. paragraph "Foo bar --- baz" +4. heading "Foo bar", paragraph "baz" + +We find interpretation 4 most natural, and interpretation 4 +increases the expressive power of CommonMark, by allowing +multiline headings. Authors who want interpretation 1 can +put a blank line after the first paragraph: + +```````````````````````````````` example +Foo + +bar +--- +baz +. +

    Foo

    +

    bar

    +

    baz

    +```````````````````````````````` + + +Authors who want interpretation 2 can put blank lines around +the thematic break, + +```````````````````````````````` example +Foo +bar + +--- + +baz +. +

    Foo +bar

    +
    +

    baz

    +```````````````````````````````` + + +or use a thematic break that cannot count as a [setext heading +underline], such as + +```````````````````````````````` example +Foo +bar +* * * +baz +. +

    Foo +bar

    +
    +

    baz

    +```````````````````````````````` + + +Authors who want interpretation 3 can use backslash escapes: + +```````````````````````````````` example +Foo +bar +\--- +baz +. +

    Foo +bar +--- +baz

    +```````````````````````````````` + + +## Indented code blocks + +An [indented code block](@) is composed of one or more +[indented chunks] separated by blank lines. +An [indented chunk](@) is a sequence of non-blank lines, +each preceded by four or more spaces of indentation. The contents of the code +block are the literal contents of the lines, including trailing +[line endings], minus four spaces of indentation. +An indented code block has no [info string]. + +An indented code block cannot interrupt a paragraph, so there must be +a blank line between a paragraph and a following indented code block. +(A blank line is not needed, however, between a code block and a following +paragraph.) + +```````````````````````````````` example + a simple + indented code block +. +
    a simple
+  indented code block
+
    +```````````````````````````````` + + +If there is any ambiguity between an interpretation of indentation +as a code block and as indicating that material belongs to a [list +item][list items], the list item interpretation takes precedence: + +```````````````````````````````` example + - foo + + bar +. +
      +
    • +

      foo

      +

      bar

      +
      • +
    +```````````````````````````````` + + +```````````````````````````````` example +1. foo + + - bar +. +
      +
    1. +

      foo

      +
        +
      • bar
        • +
      +
      2. +
    +```````````````````````````````` + + + +The contents of a code block are literal text, and do not get parsed +as Markdown: + +```````````````````````````````` example +     + *hi* + + - one +. +
    <a/>
+*hi*
+
+- one
+
    +```````````````````````````````` + + +Here we have three chunks separated by blank lines: + +```````````````````````````````` example + chunk1 + + chunk2 + + + + chunk3 +. +
    chunk1
+
+chunk2
+
+
+
+chunk3
+
    +```````````````````````````````` + + +Any initial spaces or tabs beyond four spaces of indentation will be included in +the content, even in interior blank lines: + +```````````````````````````````` example + chunk1 + + chunk2 +. +
    chunk1
+  
+  chunk2
+
    +```````````````````````````````` + + +An indented code block cannot interrupt a paragraph. (This +allows hanging indents and the like.) + +```````````````````````````````` example +Foo + bar + +. +

    Foo +bar

    +```````````````````````````````` + + +However, any non-blank line with fewer than four spaces of indentation ends +the code block immediately. So a paragraph may occur immediately +after indented code: + +```````````````````````````````` example + foo +bar +. +
    foo
+
    +

    bar

    +```````````````````````````````` + + +And indented code can occur immediately before and after other kinds of +blocks: + +```````````````````````````````` example +# Heading + foo +Heading +------ + foo +---- +. +

    Heading

    +
    foo
+
    +

    Heading

    +
    foo
+
    +
    +```````````````````````````````` + + +The first line can be preceded by more than four spaces of indentation: + +```````````````````````````````` example + foo + bar +. +
        foo
+bar
+
    +```````````````````````````````` + + +Blank lines preceding or following an indented code block +are not included in it: + +```````````````````````````````` example + + + foo + + +. +
    foo
+
    +```````````````````````````````` + + +Trailing spaces or tabs are included in the code block's content: + +```````````````````````````````` example + foo +. +
    foo  
+
    +```````````````````````````````` + + + +## Fenced code blocks + +A [code fence](@) is a sequence +of at least three consecutive backtick characters (`` ` ``) or +tildes (`~`). (Tildes and backticks cannot be mixed.) +A [fenced code block](@) +begins with a code fence, preceded by up to three spaces of indentation. + +The line with the opening code fence may optionally contain some text +following the code fence; this is trimmed of leading and trailing +spaces or tabs and called the [info string](@). If the [info string] comes +after a backtick fence, it must not contain any backtick +characters. (The reason for this restriction is that otherwise +some inline code would be incorrectly interpreted as the +beginning of a fenced code block.) + +The content of the code block consists of all subsequent lines, until +a closing [code fence] of the same type as the code block +began with (backticks or tildes), and with at least as many backticks +or tildes as the opening code fence. If the leading code fence is +preceded by N spaces of indentation, then up to N spaces of indentation are +removed from each line of the content (if present). (If a content line is not +indented, it is preserved unchanged. If it is indented N spaces or less, all +of the indentation is removed.) + +The closing code fence may be preceded by up to three spaces of indentation, and +may be followed only by spaces or tabs, which are ignored. If the end of the +containing block (or document) is reached and no closing code fence +has been found, the code block contains all of the lines after the +opening code fence until the end of the containing block (or +document). (An alternative spec would require backtracking in the +event that a closing code fence is not found. But this makes parsing +much less efficient, and there seems to be no real downside to the +behavior described here.) + +A fenced code block may interrupt a paragraph, and does not require +a blank line either before or after. + +The content of a code fence is treated as literal text, not parsed +as inlines. The first word of the [info string] is typically used to +specify the language of the code sample, and rendered in the `class` +attribute of the `code` tag. However, this spec does not mandate any +particular treatment of the [info string]. + +Here is a simple example with backticks: + +```````````````````````````````` example +``` +< + > +``` +. +
    <
+ >
+
    +```````````````````````````````` + + +With tildes: + +```````````````````````````````` example +~~~ +< + > +~~~ +. +
    <
+ >
+
    +```````````````````````````````` + +Fewer than three backticks is not enough: + +```````````````````````````````` example +`` +foo +`` +. +

    foo

    +```````````````````````````````` + +The closing code fence must use the same character as the opening +fence: + +```````````````````````````````` example +``` +aaa +~~~ +``` +. +
    aaa
+~~~
+
    +```````````````````````````````` + + +```````````````````````````````` example +~~~ +aaa +``` +~~~ +. +
    aaa
+```
+
    +```````````````````````````````` + + +The closing code fence must be at least as long as the opening fence: + +```````````````````````````````` example +```` +aaa +``` +`````` +. +
    aaa
+```
+
    +```````````````````````````````` + + +```````````````````````````````` example +~~~~ +aaa +~~~ +~~~~ +. +
    aaa
+~~~
+
    +```````````````````````````````` + + +Unclosed code blocks are closed by the end of the document +(or the enclosing [block quote][block quotes] or [list item][list items]): + +```````````````````````````````` example +``` +. +
    +```````````````````````````````` + + +```````````````````````````````` example +````` + +``` +aaa +. +
    
+```
+aaa
+
    +```````````````````````````````` + + +```````````````````````````````` example +> ``` +> aaa + +bbb +. +
    +
    aaa
+
    +
    +

    bbb

    +```````````````````````````````` + + +A code block can have all empty lines as its content: + +```````````````````````````````` example +``` + + +``` +. +
    
+  
+
    +```````````````````````````````` + + +A code block can be empty: + +```````````````````````````````` example +``` +``` +. +
    +```````````````````````````````` + + +Fences can be indented. If the opening fence is indented, +content lines will have equivalent opening indentation removed, +if present: + +```````````````````````````````` example + ``` + aaa +aaa +``` +. +
    aaa
+aaa
+
    +```````````````````````````````` + + +```````````````````````````````` example + ``` +aaa + aaa +aaa + ``` +. +
    aaa
+aaa
+aaa
+
    +```````````````````````````````` + + +```````````````````````````````` example + ``` + aaa + aaa + aaa + ``` +. +
    aaa
+ aaa
+aaa
+
    +```````````````````````````````` + + +Four spaces of indentation is too many: + +```````````````````````````````` example + ``` + aaa + ``` +. +
    ```
+aaa
+```
+
    +```````````````````````````````` + + +Closing fences may be preceded by up to three spaces of indentation, and their +indentation need not match that of the opening fence: + +```````````````````````````````` example +``` +aaa + ``` +. +
    aaa
+
    +```````````````````````````````` + + +```````````````````````````````` example + ``` +aaa + ``` +. +
    aaa
+
    +```````````````````````````````` + + +This is not a closing fence, because it is indented 4 spaces: + +```````````````````````````````` example +``` +aaa + ``` +. +
    aaa
+    ```
+
    +```````````````````````````````` + + + +Code fences (opening and closing) cannot contain internal spaces or tabs: + +```````````````````````````````` example +``` ``` +aaa +. +

    +aaa

    +```````````````````````````````` + + +```````````````````````````````` example +~~~~~~ +aaa +~~~ ~~ +. +
    aaa
+~~~ ~~
+
    +```````````````````````````````` + + +Fenced code blocks can interrupt paragraphs, and can be followed +directly by paragraphs, without a blank line between: + +```````````````````````````````` example +foo +``` +bar +``` +baz +. +

    foo

    +
    bar
+
    +

    baz

    +```````````````````````````````` + + +Other blocks can also occur before and after fenced code blocks +without an intervening blank line: + +```````````````````````````````` example +foo +--- +~~~ +bar +~~~ +# baz +. +

    foo

    +
    bar
+
    +

    baz

    +```````````````````````````````` + + +An [info string] can be provided after the opening code fence. +Although this spec doesn't mandate any particular treatment of +the info string, the first word is typically used to specify +the language of the code block. In HTML output, the language is +normally indicated by adding a class to the `code` element consisting +of `language-` followed by the language name. + +```````````````````````````````` example +```ruby +def foo(x) + return 3 +end +``` +. +
    def foo(x)
+  return 3
+end
+
    +```````````````````````````````` + + +```````````````````````````````` example +~~~~ ruby startline=3 $%@#$ +def foo(x) + return 3 +end +~~~~~~~ +. +
    def foo(x)
+  return 3
+end
+
    +```````````````````````````````` + + +```````````````````````````````` example +````; +```` +. +
    +```````````````````````````````` + + +[Info strings] for backtick code blocks cannot contain backticks: + +```````````````````````````````` example +``` aa ``` +foo +. +

    aa +foo

    +```````````````````````````````` + + +[Info strings] for tilde code blocks can contain backticks and tildes: + +```````````````````````````````` example +~~~ aa ``` ~~~ +foo +~~~ +. +
    foo
+
    +```````````````````````````````` + + +Closing code fences cannot have [info strings]: + +```````````````````````````````` example +``` +``` aaa +``` +. +
    ``` aaa
+
    +```````````````````````````````` + + + +## HTML blocks + +An [HTML block](@) is a group of lines that is treated +as raw HTML (and will not be escaped in HTML output). + +There are seven kinds of [HTML block], which can be defined by their +start and end conditions. The block begins with a line that meets a +[start condition](@) (after up to three optional spaces of indentation). +It ends with the first subsequent line that meets a matching +[end condition](@), or the last line of the document, or the last line of +the [container block](#container-blocks) containing the current HTML +block, if no line is encountered that meets the [end condition]. If +the first line meets both the [start condition] and the [end +condition], the block will contain just that line. + +1. **Start condition:** line begins with the string ``, or the end of the line.\ +**End condition:** line contains an end tag +``, ``, ``, or `` (case-insensitive; it +need not match the start tag). + +2. **Start condition:** line begins with the string ``. + +3. **Start condition:** line begins with the string ``. + +4. **Start condition:** line begins with the string ``. + +5. **Start condition:** line begins with the string +``. + +6. **Start condition:** line begins with the string `<` or ``, or +the string `/>`.\ +**End condition:** line is followed by a [blank line]. + +7. **Start condition:** line begins with a complete [open tag] +(with any [tag name] other than `pre`, `script`, +`style`, or `textarea`) or a complete [closing tag], +all on a single line, followed by zero or more spaces and tabs, +followed by the end of the line.\ +**End condition:** line is followed by a [blank line]. + +HTML blocks continue until they are closed by their appropriate +[end condition], or the last line of the document or other [container +block](#container-blocks). This means any HTML **within an HTML +block** that might otherwise be recognised as a start condition will +be ignored by the parser and passed through as-is, without changing +the parser's state. + +For instance, `
    ` within an HTML block started by `` will not affect
+the parser state; as the HTML block was started in by start condition 6, it
+will end at any blank line. This can be surprising:
+
+```````````````````````````````` example
+
    
+
    +**Hello**,
+
+_world_.
+
    
+
    
+.
+
    
+
    +**Hello**,
+
    world.
+
    
+
    
+````````````````````````````````
+
+In this case, the HTML block is terminated by the blank line — the `**Hello**`
+text remains verbatim — and regular parsing resumes, with a paragraph,
+emphasised `world` and inline and block HTML following.
+
+All types of [HTML blocks] except type 7 may interrupt
+a paragraph.  Blocks of type 7 cannot interrupt a paragraph.
+(This restriction is intended to prevent unwanted interpretation
+of long tags inside a wrapped paragraph as starting HTML blocks.)
+
+Some simple examples follow.  Here are some basic HTML blocks
+of type 6:
+
+```````````````````````````````` example
+
+  
+    
+  
+
    
+           hi
+    
    
+
+okay.
+.
+
+  
+    
+  
+
    
+           hi
+    
    
+
    okay.
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+     
+*foo*
+````````````````````````````````
+
+
+Here we have two HTML blocks with a Markdown paragraph between them:
+
+```````````````````````````````` example
+
    
+
+*Markdown*
+
+
    
+.
+
    
+
    Markdown
    
+
    
+````````````````````````````````
+
+
+The tag on the first line can be partial, as long
+as it is split where there would be whitespace:
+
+```````````````````````````````` example
+
    
+
    
+.
+
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
    
+
    
+.
+
    
+
    
+````````````````````````````````
+
+
+An open tag need not be closed:
+```````````````````````````````` example
+
    
+*foo*
+
+*bar*
+.
+
    
+*foo*
+
    bar
    
+````````````````````````````````
+
+
+
+A partial tag need not even be completed (garbage
+in, garbage out):
+
+```````````````````````````````` example
+    
+.
+
+````````````````````````````````
+
+
+```````````````````````````````` example
+
    
+foo
+
    
+.
+
    
+foo
+
    
+````````````````````````````````
+
+
+Everything until the next blank line or end of document
+gets included in the HTML block.  So, in the following
+examples, what looks like a Markdown code block or block quote
+is actually part of the HTML block, which continues until a blank
+line or the end of the document is reached:
+
+```````````````````````````````` example
+
    
+``` c
+int x = 33;
+```
+.
+
    
+``` c
+int x = 33;
+```
+````````````````````````````````
+
+
+```````````````````````````````` example
+
     not quoted text
+.
+
     not quoted text
+````````````````````````````````
+
+
+To start an [HTML block] with a tag that is *not* in the
+list of block-level tags in (6), you must put the tag by
+itself on the first line (and it must be complete):
+
+```````````````````````````````` example
+
+*bar*
+
+.
+
+*bar*
+
+````````````````````````````````
+
+
+In type 7 blocks, the [tag name] can be anything:
+
+```````````````````````````````` example
+
+*bar*
+
+.
+
+*bar*
+
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+*bar*
+
+.
+
+*bar*
+
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+*bar*
+.
+
+*bar*
+````````````````````````````````
+
+
+These rules are designed to allow us to work with tags that
+can function as either block-level or inline-level tags.
+The `` tag is a nice example.  We can surround content with
+`` tags in three different ways.  In this case, we get a raw
+HTML block, because the `` tag is on a line by itself:
+
+```````````````````````````````` example
+
+*foo*
+
+.
+
+*foo*
+
+````````````````````````````````
+
+
+In this case, we get a raw HTML block that just includes
+the `` tag (because it ends with the following blank
+line).  So the contents get interpreted as CommonMark:
+
+```````````````````````````````` example
+
+
+*foo*
+
+
+.
+
+
    foo
    
+    
+````````````````````````````````
+
+
+Finally, in these cases, the `` tags are interpreted
+as [raw HTML] *inside* the CommonMark paragraph.  (Because
+the tag is not on a line by itself, we get inline HTML
+rather than an [HTML block].)
+
+```````````````````````````````` example
+*foo*
+.
+
    foo
    
+````````````````````````````````
+
+```````````````````````````````` example
+
+*foo*
+
+.
+
    
+foo
+
    
+````````````````````````````````
+
+
+HTML tags designed to contain literal content
+(`pre`, `script`, `style`, `textarea`), comments, processing instructions,
+and declarations are treated somewhat differently.
+Instead of ending at the first blank line, these blocks
+end at the first line containing a corresponding end tag.
+As a result, these blocks can contain blank lines:
+
+A pre tag (type 1):
+
+```````````````````````````````` example
+
    
+import Text.HTML.TagSoup
+
+main :: IO ()
+main = print $ parseTags tags
+
    
+okay
+.
+
    
+import Text.HTML.TagSoup
+
+main :: IO ()
+main = print $ parseTags tags
+
    
+
    okay
    
+````````````````````````````````
+
+
+A script tag (type 1):
+
+```````````````````````````````` example
+
+okay
+.
+
+
    okay
    
+````````````````````````````````
+
+
+A textarea tag (type 1):
+
+```````````````````````````````` example
+
+.
+
+````````````````````````````````
+
+A style tag (type 1):
+
+```````````````````````````````` example
+
+okay
+.
+
+
    okay
    
+````````````````````````````````
+
+
+If there is no matching end tag, the block will end at the
+end of the document (or the enclosing [block quote][block quotes]
+or [list item][list items]):
+
+```````````````````````````````` example
+
+*foo*
+.
+
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*bar*
+*baz*
+.
+*bar*
+
    baz
    
+````````````````````````````````
+
+
+Note that anything on the last line after the
+end tag will be included in the [HTML block]:
+
+```````````````````````````````` example
+1. *bar*
+.
+1. *bar*
+````````````````````````````````
+
+
+A comment (type 2):
+
+```````````````````````````````` example
+
+okay
+.
+
+
    okay
    
+````````````````````````````````
+
+
+
+A processing instruction (type 3):
+
+```````````````````````````````` example
+';
+
+?>
+okay
+.
+';
+
+?>
+
    okay
    
+````````````````````````````````
+
+
+A declaration (type 4):
+
+```````````````````````````````` example
+
+.
+
+````````````````````````````````
+
+
+CDATA (type 5):
+
+```````````````````````````````` example
+
+okay
+.
+
+
    okay
    
+````````````````````````````````
+
+
+The opening tag can be preceded by up to three spaces of indentation, but not
+four:
+
+```````````````````````````````` example
+  
+
+    
+.
+  
+
    <!-- foo -->
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+  
    
+
+    
    
+.
+  
    
+
    <div>
+
    
+````````````````````````````````
+
+
+An HTML block of types 1--6 can interrupt a paragraph, and need not be
+preceded by a blank line.
+
+```````````````````````````````` example
+Foo
+
    
+bar
+
    
+.
+
    Foo
    
+
    
+bar
+
    
+````````````````````````````````
+
+
+However, a following blank line is needed, except at the end of
+a document, and except for blocks of types 1--5, [above][HTML
+block]:
+
+```````````````````````````````` example
+
    
+bar
+
    
+*foo*
+.
+
    
+bar
+
    
+*foo*
+````````````````````````````````
+
+
+HTML blocks of type 7 cannot interrupt a paragraph:
+
+```````````````````````````````` example
+Foo
+
+baz
+.
+
    Foo
+
+baz
    
+````````````````````````````````
+
+
+This rule differs from John Gruber's original Markdown syntax
+specification, which says:
+
+> The only restrictions are that block-level HTML elements —
+> e.g. `
    `, ``, `
    `, `
    `, etc. — must be separated from
+> surrounding content by blank lines, and the start and end tags of the
+> block should not be indented with spaces or tabs.
+
+In some ways Gruber's rule is more restrictive than the one given
+here:
+
+- It requires that an HTML block be preceded by a blank line.
+- It does not allow the start tag to be indented.
+- It requires a matching end tag, which it also does not allow to
+  be indented.
+
+Most Markdown implementations (including some of Gruber's own) do not
+respect all of these restrictions.
+
+There is one respect, however, in which Gruber's rule is more liberal
+than the one given here, since it allows blank lines to occur inside
+an HTML block.  There are two reasons for disallowing them here.
+First, it removes the need to parse balanced tags, which is
+expensive and can require backtracking from the end of the document
+if no matching end tag is found. Second, it provides a very simple
+and flexible way of including Markdown content inside HTML tags:
+simply separate the Markdown from the HTML using blank lines:
+
+Compare:
+
+```````````````````````````````` example
+
    
+
+*Emphasized* text.
+
+
    
+.
+
    
+
    Emphasized text.
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
    
+*Emphasized* text.
+
    
+.
+
    
+*Emphasized* text.
+
    
+````````````````````````````````
+
+
+Some Markdown implementations have adopted a convention of
+interpreting content inside tags as text if the open tag has
+the attribute `markdown=1`.  The rule given above seems a simpler and
+more elegant way of achieving the same expressive power, which is also
+much simpler to parse.
+
+The main potential drawback is that one can no longer paste HTML
+blocks into Markdown documents with 100% reliability.  However,
+*in most cases* this will work fine, because the blank lines in
+HTML are usually followed by HTML block tags.  For example:
+
+```````````````````````````````` example
+
    
+
+
+
+
+
+
+
+
    
+Hi
+
    
+.
+
+
+
+
+
    
+Hi
+
    
+````````````````````````````````
+
+
+There are problems, however, if the inner tags are indented
+*and* separated by spaces, as then they will be interpreted as
+an indented code block:
+
+```````````````````````````````` example
+
+
+  
+
+    
+
+  
+
+
    
+      Hi
+    
    
+.
+
+  
+
    <td>
+  Hi
+</td>
+
    
+  
+
    
+````````````````````````````````
+
+
+Fortunately, blank lines are usually not necessary and can be
+deleted.  The exception is inside `
    ` tags, but as described
+[above][HTML blocks], raw HTML blocks starting with `
    `
+*can* contain blank lines.
+
+## Link reference definitions
+
+A [link reference definition](@)
+consists of a [link label], optionally preceded by up to three spaces of
+indentation, followed
+by a colon (`:`), optional spaces or tabs (including up to one
+[line ending]), a [link destination],
+optional spaces or tabs (including up to one
+[line ending]), and an optional [link
+title], which if it is present must be separated
+from the [link destination] by spaces or tabs.
+No further character may occur.
+
+A [link reference definition]
+does not correspond to a structural element of a document.  Instead, it
+defines a label which can be used in [reference links]
+and reference-style [images] elsewhere in the document.  [Link
+reference definitions] can come either before or after the links that use
+them.
+
+```````````````````````````````` example
+[foo]: /url "title"
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+   [foo]: 
+      /url  
+           'the title'  
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[Foo*bar\]]:my_(url) 'title (with parens)'
+
+[Foo*bar\]]
+.
+
    Foo*bar]
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[Foo bar]:
+
+'title'
+
+[Foo bar]
+.
+
    Foo bar
    
+````````````````````````````````
+
+
+The title may extend over multiple lines:
+
+```````````````````````````````` example
+[foo]: /url '
+title
+line1
+line2
+'
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+
+However, it must not contain a [blank line]:
+
+```````````````````````````````` example
+[foo]: /url 'title
+
+with blank line'
+
+[foo]
+.
+
    [foo]: /url 'title
    
+
    with blank line'
    
+
    [foo]
    
+````````````````````````````````
+
+
+The title may be omitted:
+
+```````````````````````````````` example
+[foo]:
+/url
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+
+The link destination must not be omitted:
+
+```````````````````````````````` example
+[foo]:
+
+[foo]
+.
+
    [foo]:
    
+
    [foo]
    
+````````````````````````````````
+
+ However, an empty link destination may be specified using
+ angle brackets:
+
+```````````````````````````````` example
+[foo]: <>
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+The title must be separated from the link destination by
+spaces or tabs:
+
+```````````````````````````````` example
+[foo]: (baz)
+
+[foo]
+.
+
    [foo]: (baz)
    
+
    [foo]
    
+````````````````````````````````
+
+
+Both title and destination can contain backslash escapes
+and literal backslashes:
+
+```````````````````````````````` example
+[foo]: /url\bar\*baz "foo\"bar\baz"
+
+[foo]
+.
+
    foo
    
+````````````````````````````````
+
+
+A link can come before its corresponding definition:
+
+```````````````````````````````` example
+[foo]
+
+[foo]: url
+.
+
    foo
    
+````````````````````````````````
+
+
+If there are several matching definitions, the first one takes
+precedence:
+
+```````````````````````````````` example
+[foo]
+
+[foo]: first
+[foo]: second
+.
+
    foo
    
+````````````````````````````````
+
+
+As noted in the section on [Links], matching of labels is
+case-insensitive (see [matches]).
+
+```````````````````````````````` example
+[FOO]: /url
+
+[Foo]
+.
+
    Foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[ΑΓΩ]: /φου
+
+[αγω]
+.
+
    αγω
    
+````````````````````````````````
+
+
+Whether something is a [link reference definition] is
+independent of whether the link reference it defines is
+used in the document.  Thus, for example, the following
+document contains just a link reference definition, and
+no visible content:
+
+```````````````````````````````` example
+[foo]: /url
+.
+````````````````````````````````
+
+
+Here is another one:
+
+```````````````````````````````` example
+[
+foo
+]: /url
+bar
+.
+
    bar
    
+````````````````````````````````
+
+
+This is not a link reference definition, because there are
+characters other than spaces or tabs after the title:
+
+```````````````````````````````` example
+[foo]: /url "title" ok
+.
+
    [foo]: /url "title" ok
    
+````````````````````````````````
+
+
+This is a link reference definition, but it has no title:
+
+```````````````````````````````` example
+[foo]: /url
+"title" ok
+.
+
    "title" ok
    
+````````````````````````````````
+
+
+This is not a link reference definition, because it is indented
+four spaces:
+
+```````````````````````````````` example
+    [foo]: /url "title"
+
+[foo]
+.
+
    [foo]: /url "title"
+
    
+
    [foo]
    
+````````````````````````````````
+
+
+This is not a link reference definition, because it occurs inside
+a code block:
+
+```````````````````````````````` example
+```
+[foo]: /url
+```
+
+[foo]
+.
+
    [foo]: /url
+
    
+
    [foo]
    
+````````````````````````````````
+
+
+A [link reference definition] cannot interrupt a paragraph.
+
+```````````````````````````````` example
+Foo
+[bar]: /baz
+
+[bar]
+.
+
    Foo
+[bar]: /baz
    
+
    [bar]
    
+````````````````````````````````
+
+
+However, it can directly follow other block elements, such as headings
+and thematic breaks, and it need not be followed by a blank line.
+
+```````````````````````````````` example
+# [Foo]
+[foo]: /url
+> bar
+.
+
    Foo
    
+
    
+
    bar
    
+
    
+````````````````````````````````
+
+```````````````````````````````` example
+[foo]: /url
+bar
+===
+[foo]
+.
+
    bar
    
+
    foo
    
+````````````````````````````````
+
+```````````````````````````````` example
+[foo]: /url
+===
+[foo]
+.
+
    ===
+foo
    
+````````````````````````````````
+
+
+Several [link reference definitions]
+can occur one after another, without intervening blank lines.
+
+```````````````````````````````` example
+[foo]: /foo-url "foo"
+[bar]: /bar-url
+  "bar"
+[baz]: /baz-url
+
+[foo],
+[bar],
+[baz]
+.
+
    foo,
+bar,
+baz
    
+````````````````````````````````
+
+
+[Link reference definitions] can occur
+inside block containers, like lists and block quotations.  They
+affect the entire document, not just the container in which they
+are defined:
+
+```````````````````````````````` example
+[foo]
+
+> [foo]: /url
+.
+
    foo
    
+
    
+
    
+````````````````````````````````
+
+
+## Paragraphs
+
+A sequence of non-blank lines that cannot be interpreted as other
+kinds of blocks forms a [paragraph](@).
+The contents of the paragraph are the result of parsing the
+paragraph's raw content as inlines.  The paragraph's raw content
+is formed by concatenating the lines and removing initial and final
+spaces or tabs.
+
+A simple example with two paragraphs:
+
+```````````````````````````````` example
+aaa
+
+bbb
+.
+
    aaa
    
+
    bbb
    
+````````````````````````````````
+
+
+Paragraphs can contain multiple lines, but no blank lines:
+
+```````````````````````````````` example
+aaa
+bbb
+
+ccc
+ddd
+.
+
    aaa
+bbb
    
+
    ccc
+ddd
    
+````````````````````````````````
+
+
+Multiple blank lines between paragraphs have no effect:
+
+```````````````````````````````` example
+aaa
+
+
+bbb
+.
+
    aaa
    
+
    bbb
    
+````````````````````````````````
+
+
+Leading spaces or tabs are skipped:
+
+```````````````````````````````` example
+  aaa
+ bbb
+.
+
    aaa
+bbb
    
+````````````````````````````````
+
+
+Lines after the first may be indented any amount, since indented
+code blocks cannot interrupt paragraphs.
+
+```````````````````````````````` example
+aaa
+             bbb
+                                       ccc
+.
+
    aaa
+bbb
+ccc
    
+````````````````````````````````
+
+
+However, the first line may be preceded by up to three spaces of indentation.
+Four spaces of indentation is too many:
+
+```````````````````````````````` example
+   aaa
+bbb
+.
+
    aaa
+bbb
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+    aaa
+bbb
+.
+
    aaa
+
    
+
    bbb
    
+````````````````````````````````
+
+
+Final spaces or tabs are stripped before inline parsing, so a paragraph
+that ends with two or more spaces will not end with a [hard line
+break]:
+
+```````````````````````````````` example
+aaa     
+bbb     
+.
+
    aaa
    
+bbb
    
+````````````````````````````````
+
+
+## Blank lines
+
+[Blank lines] between block-level elements are ignored,
+except for the role they play in determining whether a [list]
+is [tight] or [loose].
+
+Blank lines at the beginning and end of the document are also ignored.
+
+```````````````````````````````` example
+  
+
+aaa
+  
+
+# aaa
+
+  
+.
+
    aaa
    
+
    aaa
    
+````````````````````````````````
+
+
+
+# Container blocks
+
+A [container block](#container-blocks) is a block that has other
+blocks as its contents.  There are two basic kinds of container blocks:
+[block quotes] and [list items].
+[Lists] are meta-containers for [list items].
+
+We define the syntax for container blocks recursively.  The general
+form of the definition is:
+
+> If X is a sequence of blocks, then the result of
+> transforming X in such-and-such a way is a container of type Y
+> with these blocks as its content.
+
+So, we explain what counts as a block quote or list item by explaining
+how these can be *generated* from their contents. This should suffice
+to define the syntax, although it does not give a recipe for *parsing*
+these constructions.  (A recipe is provided below in the section entitled
+[A parsing strategy](#appendix-a-parsing-strategy).)
+
+## Block quotes
+
+A [block quote marker](@),
+optionally preceded by up to three spaces of indentation,
+consists of (a) the character `>` together with a following space of
+indentation, or (b) a single character `>` not followed by a space of
+indentation.
+
+The following rules define [block quotes]:
+
+1.  **Basic case.**  If a string of lines *Ls* constitute a sequence
+    of blocks *Bs*, then the result of prepending a [block quote
+    marker] to the beginning of each line in *Ls*
+    is a [block quote](#block-quotes) containing *Bs*.
+
+2.  **Laziness.**  If a string of lines *Ls* constitute a [block
+    quote](#block-quotes) with contents *Bs*, then the result of deleting
+    the initial [block quote marker] from one or
+    more lines in which the next character other than a space or tab after the
+    [block quote marker] is [paragraph continuation
+    text] is a block quote with *Bs* as its content.
+    [Paragraph continuation text](@) is text
+    that will be parsed as part of the content of a paragraph, but does
+    not occur at the beginning of the paragraph.
+
+3.  **Consecutiveness.**  A document cannot contain two [block
+    quotes] in a row unless there is a [blank line] between them.
+
+Nothing else counts as a [block quote](#block-quotes).
+
+Here is a simple example:
+
+```````````````````````````````` example
+> # Foo
+> bar
+> baz
+.
+
    
+
    Foo
    
+
    bar
+baz
    
+
    
+````````````````````````````````
+
+
+The space or tab after the `>` characters can be omitted:
+
+```````````````````````````````` example
+># Foo
+>bar
+> baz
+.
+
    
+
    Foo
    
+
    bar
+baz
    
+
    
+````````````````````````````````
+
+
+The `>` characters can be preceded by up to three spaces of indentation:
+
+```````````````````````````````` example
+   > # Foo
+   > bar
+ > baz
+.
+
    
+
    Foo
    
+
    bar
+baz
    
+
    
+````````````````````````````````
+
+
+Four spaces of indentation is too many:
+
+```````````````````````````````` example
+    > # Foo
+    > bar
+    > baz
+.
+
    > # Foo
+> bar
+> baz
+
    
+````````````````````````````````
+
+
+The Laziness clause allows us to omit the `>` before
+[paragraph continuation text]:
+
+```````````````````````````````` example
+> # Foo
+> bar
+baz
+.
+
    
+
    Foo
    
+
    bar
+baz
    
+
    
+````````````````````````````````
+
+
+A block quote can contain some lazy and some non-lazy
+continuation lines:
+
+```````````````````````````````` example
+> bar
+baz
+> foo
+.
+
    
+
    bar
+baz
+foo
    
+
    
+````````````````````````````````
+
+
+Laziness only applies to lines that would have been continuations of
+paragraphs had they been prepended with [block quote markers].
+For example, the `> ` cannot be omitted in the second line of
+
+``` markdown
+> foo
+> ---
+```
+
+without changing the meaning:
+
+```````````````````````````````` example
+> foo
+---
+.
+
    
+
    foo
    
+
    
+
    
+````````````````````````````````
+
+
+Similarly, if we omit the `> ` in the second line of
+
+``` markdown
+> - foo
+> - bar
+```
+
+then the block quote ends after the first line:
+
+```````````````````````````````` example
+> - foo
+- bar
+.
+
    
+
      
+
    • foo
      • 
+
    
+
    
+
      
+
    • bar
      • 
+
    
+````````````````````````````````
+
+
+For the same reason, we can't omit the `> ` in front of
+subsequent lines of an indented or fenced code block:
+
+```````````````````````````````` example
+>     foo
+    bar
+.
+
    
+
    foo
+
    
+
    
+
    bar
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+> ```
+foo
+```
+.
+
    
+
    
+
    
+
    foo
    
+
    
+````````````````````````````````
+
+
+Note that in the following case, we have a [lazy
+continuation line]:
+
+```````````````````````````````` example
+> foo
+    - bar
+.
+
    
+
    foo
+- bar
    
+
    
+````````````````````````````````
+
+
+To see why, note that in
+
+```markdown
+> foo
+>     - bar
+```
+
+the `- bar` is indented too far to start a list, and can't
+be an indented code block because indented code blocks cannot
+interrupt paragraphs, so it is [paragraph continuation text].
+
+A block quote can be empty:
+
+```````````````````````````````` example
+>
+.
+
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+>
+>  
+> 
+.
+
    
+
    
+````````````````````````````````
+
+
+A block quote can have initial or final blank lines:
+
+```````````````````````````````` example
+>
+> foo
+>  
+.
+
    
+
    foo
    
+
    
+````````````````````````````````
+
+
+A blank line always separates block quotes:
+
+```````````````````````````````` example
+> foo
+
+> bar
+.
+
    
+
    foo
    
+
    
+
    
+
    bar
    
+
    
+````````````````````````````````
+
+
+(Most current Markdown implementations, including John Gruber's
+original `Markdown.pl`, will parse this example as a single block quote
+with two paragraphs.  But it seems better to allow the author to decide
+whether two block quotes or one are wanted.)
+
+Consecutiveness means that if we put these block quotes together,
+we get a single block quote:
+
+```````````````````````````````` example
+> foo
+> bar
+.
+
    
+
    foo
+bar
    
+
    
+````````````````````````````````
+
+
+To get a block quote with two paragraphs, use:
+
+```````````````````````````````` example
+> foo
+>
+> bar
+.
+
    
+
    foo
    
+
    bar
    
+
    
+````````````````````````````````
+
+
+Block quotes can interrupt paragraphs:
+
+```````````````````````````````` example
+foo
+> bar
+.
+
    foo
    
+
    
+
    bar
    
+
    
+````````````````````````````````
+
+
+In general, blank lines are not needed before or after block
+quotes:
+
+```````````````````````````````` example
+> aaa
+***
+> bbb
+.
+
    
+
    aaa
    
+
    
+
    
+
    
+
    bbb
    
+
    
+````````````````````````````````
+
+
+However, because of laziness, a blank line is needed between
+a block quote and a following paragraph:
+
+```````````````````````````````` example
+> bar
+baz
+.
+
    
+
    bar
+baz
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+> bar
+
+baz
+.
+
    
+
    bar
    
+
    
+
    baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+> bar
+>
+baz
+.
+
    
+
    bar
    
+
    
+
    baz
    
+````````````````````````````````
+
+
+It is a consequence of the Laziness rule that any number
+of initial `>`s may be omitted on a continuation line of a
+nested block quote:
+
+```````````````````````````````` example
+> > > foo
+bar
+.
+
    
+
    
+
    
+
    foo
+bar
    
+
    
+
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+>>> foo
+> bar
+>>baz
+.
+
    
+
    
+
    
+
    foo
+bar
+baz
    
+
    
+
    
+
    
+````````````````````````````````
+
+
+When including an indented code block in a block quote,
+remember that the [block quote marker] includes
+both the `>` and a following space of indentation.  So *five spaces* are needed
+after the `>`:
+
+```````````````````````````````` example
+>     code
+
+>    not code
+.
+
    
+
    code
+
    
+
    
+
    
+
    not code
    
+
    
+````````````````````````````````
+
+
+
+## List items
+
+A [list marker](@) is a
+[bullet list marker] or an [ordered list marker].
+
+A [bullet list marker](@)
+is a `-`, `+`, or `*` character.
+
+An [ordered list marker](@)
+is a sequence of 1--9 arabic digits (`0-9`), followed by either a
+`.` character or a `)` character.  (The reason for the length
+limit is that with 10 digits we start seeing integer overflows
+in some browsers.)
+
+The following rules define [list items]:
+
+1.  **Basic case.**  If a sequence of lines *Ls* constitute a sequence of
+    blocks *Bs* starting with a character other than a space or tab, and *M* is
+    a list marker of width *W* followed by 1 ≤ *N* ≤ 4 spaces of indentation,
+    then the result of prepending *M* and the following spaces to the first line
+    of *Ls*, and indenting subsequent lines of *Ls* by *W + N* spaces, is a
+    list item with *Bs* as its contents.  The type of the list item
+    (bullet or ordered) is determined by the type of its list marker.
+    If the list item is ordered, then it is also assigned a start
+    number, based on the ordered list marker.
+
+    Exceptions:
+
+    1. When the first list item in a [list] interrupts
+       a paragraph---that is, when it starts on a line that would
+       otherwise count as [paragraph continuation text]---then (a)
+       the lines *Ls* must not begin with a blank line, and (b) if
+       the list item is ordered, the start number must be 1.
+    2. If any line is a [thematic break][thematic breaks] then
+       that line is not a list item.
+
+For example, let *Ls* be the lines
+
+```````````````````````````````` example
+A paragraph
+with two lines.
+
+    indented code
+
+> A block quote.
+.
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+````````````````````````````````
+
+
+And let *M* be the marker `1.`, and *N* = 2.  Then rule #1 says
+that the following is an ordered list item with start number 1,
+and the same contents as *Ls*:
+
+```````````````````````````````` example
+1.  A paragraph
+    with two lines.
+
+        indented code
+
+    > A block quote.
+.
+
      
+
    1. 
+
      A paragraph
+with two lines.
      
+
      indented code
+
      
+
      
+
      A block quote.
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+The most important thing to notice is that the position of
+the text after the list marker determines how much indentation
+is needed in subsequent blocks in the list item.  If the list
+marker takes up two spaces of indentation, and there are three spaces between
+the list marker and the next character other than a space or tab, then blocks
+must be indented five spaces in order to fall under the list
+item.
+
+Here are some examples showing how far content must be indented to be
+put under the list item:
+
+```````````````````````````````` example
+- one
+
+ two
+.
+
      
+
    • one
      • 
+
    
+
    two
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+- one
+
+  two
+.
+
      
+
    • 
+
      one
      
+
      two
      
+
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+ -    one
+
+     two
+.
+
      
+
    • one
      • 
+
    
+
     two
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+ -    one
+
+      two
+.
+
      
+
    • 
+
      one
      
+
      two
      
+
      • 
+
    
+````````````````````````````````
+
+
+It is tempting to think of this in terms of columns:  the continuation
+blocks must be indented at least to the column of the first character other than
+a space or tab after the list marker.  However, that is not quite right.
+The spaces of indentation after the list marker determine how much relative
+indentation is needed.  Which column this indentation reaches will depend on
+how the list item is embedded in other constructions, as shown by
+this example:
+
+```````````````````````````````` example
+   > > 1.  one
+>>
+>>     two
+.
+
    
+
    
+
      
+
    1. 
+
      one
      
+
      two
      
+
      2. 
+
    
+
    
+
    
+````````````````````````````````
+
+
+Here `two` occurs in the same column as the list marker `1.`,
+but is actually contained in the list item, because there is
+sufficient indentation after the last containing blockquote marker.
+
+The converse is also possible.  In the following example, the word `two`
+occurs far to the right of the initial text of the list item, `one`, but
+it is not considered part of the list item, because it is not indented
+far enough past the blockquote marker:
+
+```````````````````````````````` example
+>>- one
+>>
+  >  > two
+.
+
    
+
    
+
      
+
    • one
      • 
+
    
+
    two
    
+
    
+
    
+````````````````````````````````
+
+
+Note that at least one space or tab is needed between the list marker and
+any following content, so these are not list items:
+
+```````````````````````````````` example
+-one
+
+2.two
+.
+
    -one
    
+
    2.two
    
+````````````````````````````````
+
+
+A list item may contain blocks that are separated by more than
+one blank line.
+
+```````````````````````````````` example
+- foo
+
+
+  bar
+.
+
      
+
    • 
+
      foo
      
+
      bar
      
+
      • 
+
    
+````````````````````````````````
+
+
+A list item may contain any kind of block:
+
+```````````````````````````````` example
+1.  foo
+
+    ```
+    bar
+    ```
+
+    baz
+
+    > bam
+.
+
      
+
    1. 
+
      foo
      
+
      bar
+
      
+
      baz
      
+
      
+
      bam
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+A list item that contains an indented code block will preserve
+empty lines within the code block verbatim.
+
+```````````````````````````````` example
+- Foo
+
+      bar
+
+
+      baz
+.
+
      
+
    • 
+
      Foo
      
+
      bar
+
+
+baz
+
      
+
      • 
+
    
+````````````````````````````````
+
+Note that ordered list start numbers must be nine digits or less:
+
+```````````````````````````````` example
+123456789. ok
+.
+
      
+
    1. ok
      2. 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+1234567890. not ok
+.
+
    1234567890. not ok
    
+````````````````````````````````
+
+
+A start number may begin with 0s:
+
+```````````````````````````````` example
+0. ok
+.
+
      
+
    1. ok
      2. 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+003. ok
+.
+
      
+
    1. ok
      2. 
+
    
+````````````````````````````````
+
+
+A start number must not be negative:
+
+```````````````````````````````` example
+-1. not ok
+.
+
    -1. not ok
    
+````````````````````````````````
+
+
+
+2.  **Item starting with indented code.**  If a sequence of lines *Ls*
+    constitute a sequence of blocks *Bs* starting with an indented code
+    block, and *M* is a list marker of width *W* followed by
+    one space of indentation, then the result of prepending *M* and the
+    following space to the first line of *Ls*, and indenting subsequent lines
+    of *Ls* by *W + 1* spaces, is a list item with *Bs* as its contents.
+    If a line is empty, then it need not be indented.  The type of the
+    list item (bullet or ordered) is determined by the type of its list
+    marker.  If the list item is ordered, then it is also assigned a
+    start number, based on the ordered list marker.
+
+An indented code block will have to be preceded by four spaces of indentation
+beyond the edge of the region where text will be included in the list item.
+In the following case that is 6 spaces:
+
+```````````````````````````````` example
+- foo
+
+      bar
+.
+
      
+
    • 
+
      foo
      
+
      bar
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+And in this case it is 11 spaces:
+
+```````````````````````````````` example
+  10.  foo
+
+           bar
+.
+
      
+
    1. 
+
      foo
      
+
      bar
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+If the *first* block in the list item is an indented code block,
+then by rule #2, the contents must be preceded by *one* space of indentation
+after the list marker:
+
+```````````````````````````````` example
+    indented code
+
+paragraph
+
+    more code
+.
+
    indented code
+
    
+
    paragraph
    
+
    more code
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+1.     indented code
+
+   paragraph
+
+       more code
+.
+
      
+
    1. 
+
      indented code
+
      
+
      paragraph
      
+
      more code
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Note that an additional space of indentation is interpreted as space
+inside the code block:
+
+```````````````````````````````` example
+1.      indented code
+
+   paragraph
+
+       more code
+.
+
      
+
    1. 
+
       indented code
+
      
+
      paragraph
      
+
      more code
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Note that rules #1 and #2 only apply to two cases:  (a) cases
+in which the lines to be included in a list item begin with a
+character other than a space or tab, and (b) cases in which
+they begin with an indented code
+block.  In a case like the following, where the first block begins with
+three spaces of indentation, the rules do not allow us to form a list item by
+indenting the whole thing and prepending a list marker:
+
+```````````````````````````````` example
+   foo
+
+bar
+.
+
    foo
    
+
    bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+-    foo
+
+  bar
+.
+
      
+
    • foo
      • 
+
    
+
    bar
    
+````````````````````````````````
+
+
+This is not a significant restriction, because when a block is preceded by up to
+three spaces of indentation, the indentation can always be removed without
+a change in interpretation, allowing rule #1 to be applied.  So, in
+the above case:
+
+```````````````````````````````` example
+-  foo
+
+   bar
+.
+
      
+
    • 
+
      foo
      
+
      bar
      
+
      • 
+
    
+````````````````````````````````
+
+
+3.  **Item starting with a blank line.**  If a sequence of lines *Ls*
+    starting with a single [blank line] constitute a (possibly empty)
+    sequence of blocks *Bs*, and *M* is a list marker of width *W*,
+    then the result of prepending *M* to the first line of *Ls*, and
+    preceding subsequent lines of *Ls* by *W + 1* spaces of indentation, is a
+    list item with *Bs* as its contents.
+    If a line is empty, then it need not be indented.  The type of the
+    list item (bullet or ordered) is determined by the type of its list
+    marker.  If the list item is ordered, then it is also assigned a
+    start number, based on the ordered list marker.
+
+Here are some list items that start with a blank line but are not empty:
+
+```````````````````````````````` example
+-
+  foo
+-
+  ```
+  bar
+  ```
+-
+      baz
+.
+
      
+
    • foo
      • 
+
    • 
+
      bar
+
      
+
      • 
+
    • 
+
      baz
+
      
+
      • 
+
    
+````````````````````````````````
+
+When the list item starts with a blank line, the number of spaces
+following the list marker doesn't change the required indentation:
+
+```````````````````````````````` example
+-   
+  foo
+.
+
      
+
    • foo
      • 
+
    
+````````````````````````````````
+
+
+A list item can begin with at most one blank line.
+In the following example, `foo` is not part of the list
+item:
+
+```````````````````````````````` example
+-
+
+  foo
+.
+
      
+
    • 
+
    
+
    foo
    
+````````````````````````````````
+
+
+Here is an empty bullet list item:
+
+```````````````````````````````` example
+- foo
+-
+- bar
+.
+
      
+
    • foo
      • 
+
    • 
+
    • bar
      • 
+
    
+````````````````````````````````
+
+
+It does not matter whether there are spaces or tabs following the [list marker]:
+
+```````````````````````````````` example
+- foo
+-   
+- bar
+.
+
      
+
    • foo
      • 
+
    • 
+
    • bar
      • 
+
    
+````````````````````````````````
+
+
+Here is an empty ordered list item:
+
+```````````````````````````````` example
+1. foo
+2.
+3. bar
+.
+
      
+
    1. foo
      2. 
+
    2. 
+
    3. bar
      4. 
+
    
+````````````````````````````````
+
+
+A list may start or end with an empty list item:
+
+```````````````````````````````` example
+*
+.
+
      
+
    • 
+
    
+````````````````````````````````
+
+However, an empty list item cannot interrupt a paragraph:
+
+```````````````````````````````` example
+foo
+*
+
+foo
+1.
+.
+
    foo
+*
    
+
    foo
+1.
    
+````````````````````````````````
+
+
+4.  **Indentation.**  If a sequence of lines *Ls* constitutes a list item
+    according to rule #1, #2, or #3, then the result of preceding each line
+    of *Ls* by up to three spaces of indentation (the same for each line) also
+    constitutes a list item with the same contents and attributes.  If a line is
+    empty, then it need not be indented.
+
+Indented one space:
+
+```````````````````````````````` example
+ 1.  A paragraph
+     with two lines.
+
+         indented code
+
+     > A block quote.
+.
+
      
+
    1. 
+
      A paragraph
+with two lines.
      
+
      indented code
+
      
+
      
+
      A block quote.
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Indented two spaces:
+
+```````````````````````````````` example
+  1.  A paragraph
+      with two lines.
+
+          indented code
+
+      > A block quote.
+.
+
      
+
    1. 
+
      A paragraph
+with two lines.
      
+
      indented code
+
      
+
      
+
      A block quote.
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Indented three spaces:
+
+```````````````````````````````` example
+   1.  A paragraph
+       with two lines.
+
+           indented code
+
+       > A block quote.
+.
+
      
+
    1. 
+
      A paragraph
+with two lines.
      
+
      indented code
+
      
+
      
+
      A block quote.
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Four spaces indent gives a code block:
+
+```````````````````````````````` example
+    1.  A paragraph
+        with two lines.
+
+            indented code
+
+        > A block quote.
+.
+
    1.  A paragraph
+    with two lines.
+
+        indented code
+
+    > A block quote.
+
    
+````````````````````````````````
+
+
+
+5.  **Laziness.**  If a string of lines *Ls* constitute a [list
+    item](#list-items) with contents *Bs*, then the result of deleting
+    some or all of the indentation from one or more lines in which the
+    next character other than a space or tab after the indentation is
+    [paragraph continuation text] is a
+    list item with the same contents and attributes.  The unindented
+    lines are called
+    [lazy continuation line](@)s.
+
+Here is an example with [lazy continuation lines]:
+
+```````````````````````````````` example
+  1.  A paragraph
+with two lines.
+
+          indented code
+
+      > A block quote.
+.
+
      
+
    1. 
+
      A paragraph
+with two lines.
      
+
      indented code
+
      
+
      
+
      A block quote.
      
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Indentation can be partially deleted:
+
+```````````````````````````````` example
+  1.  A paragraph
+    with two lines.
+.
+
      
+
    1. A paragraph
+with two lines.
      2. 
+
    
+````````````````````````````````
+
+
+These examples show how laziness can work in nested structures:
+
+```````````````````````````````` example
+> 1. > Blockquote
+continued here.
+.
+
    
+
      
+
    1. 
+
      
+
      Blockquote
+continued here.
      
+
      
+
      2. 
+
    
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+> 1. > Blockquote
+> continued here.
+.
+
    
+
      
+
    1. 
+
      
+
      Blockquote
+continued here.
      
+
      
+
      2. 
+
    
+
    
+````````````````````````````````
+
+
+
+6.  **That's all.** Nothing that is not counted as a list item by rules
+    #1--5 counts as a [list item](#list-items).
+
+The rules for sublists follow from the general rules
+[above][List items].  A sublist must be indented the same number
+of spaces of indentation a paragraph would need to be in order to be included
+in the list item.
+
+So, in this case we need two spaces indent:
+
+```````````````````````````````` example
+- foo
+  - bar
+    - baz
+      - boo
+.
+
      
+
    • foo
+
        
+
      • bar
+
          
+
        • baz
+
            
+
          • boo
            • 
+
          
+
          • 
+
        
+
        • 
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+One is not enough:
+
+```````````````````````````````` example
+- foo
+ - bar
+  - baz
+   - boo
+.
+
      
+
    • foo
      • 
+
    • bar
      • 
+
    • baz
      • 
+
    • boo
      • 
+
    
+````````````````````````````````
+
+
+Here we need four, because the list marker is wider:
+
+```````````````````````````````` example
+10) foo
+    - bar
+.
+
      
+
    1. foo
+
        
+
      • bar
        • 
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Three is not enough:
+
+```````````````````````````````` example
+10) foo
+   - bar
+.
+
      
+
    1. foo
      2. 
+
    
+
      
+
    • bar
      • 
+
    
+````````````````````````````````
+
+
+A list may be the first block in a list item:
+
+```````````````````````````````` example
+- - foo
+.
+
      
+
    • 
+
        
+
      • foo
        • 
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+1. - 2. foo
+.
+
      
+
    1. 
+
        
+
      • 
+
          
+
        1. foo
          2. 
+
        
+
        • 
+
      
+
      2. 
+
    
+````````````````````````````````
+
+
+A list item can contain a heading:
+
+```````````````````````````````` example
+- # Foo
+- Bar
+  ---
+  baz
+.
+
      
+
    • 
+
      Foo
      
+
      • 
+
    • 
+
      Bar
      
+baz
      • 
+
    
+````````````````````````````````
+
+
+### Motivation
+
+John Gruber's Markdown spec says the following about list items:
+
+1. "List markers typically start at the left margin, but may be indented
+   by up to three spaces. List markers must be followed by one or more
+   spaces or a tab."
+
+2. "To make lists look nice, you can wrap items with hanging indents....
+   But if you don't want to, you don't have to."
+
+3. "List items may consist of multiple paragraphs. Each subsequent
+   paragraph in a list item must be indented by either 4 spaces or one
+   tab."
+
+4. "It looks nice if you indent every line of the subsequent paragraphs,
+   but here again, Markdown will allow you to be lazy."
+
+5. "To put a blockquote within a list item, the blockquote's `>`
+   delimiters need to be indented."
+
+6. "To put a code block within a list item, the code block needs to be
+   indented twice — 8 spaces or two tabs."
+
+These rules specify that a paragraph under a list item must be indented
+four spaces (presumably, from the left margin, rather than the start of
+the list marker, but this is not said), and that code under a list item
+must be indented eight spaces instead of the usual four.  They also say
+that a block quote must be indented, but not by how much; however, the
+example given has four spaces indentation.  Although nothing is said
+about other kinds of block-level content, it is certainly reasonable to
+infer that *all* block elements under a list item, including other
+lists, must be indented four spaces.  This principle has been called the
+*four-space rule*.
+
+The four-space rule is clear and principled, and if the reference
+implementation `Markdown.pl` had followed it, it probably would have
+become the standard.  However, `Markdown.pl` allowed paragraphs and
+sublists to start with only two spaces indentation, at least on the
+outer level.  Worse, its behavior was inconsistent: a sublist of an
+outer-level list needed two spaces indentation, but a sublist of this
+sublist needed three spaces.  It is not surprising, then, that different
+implementations of Markdown have developed very different rules for
+determining what comes under a list item.  (Pandoc and python-Markdown,
+for example, stuck with Gruber's syntax description and the four-space
+rule, while discount, redcarpet, marked, PHP Markdown, and others
+followed `Markdown.pl`'s behavior more closely.)
+
+Unfortunately, given the divergences between implementations, there
+is no way to give a spec for list items that will be guaranteed not
+to break any existing documents.  However, the spec given here should
+correctly handle lists formatted with either the four-space rule or
+the more forgiving `Markdown.pl` behavior, provided they are laid out
+in a way that is natural for a human to read.
+
+The strategy here is to let the width and indentation of the list marker
+determine the indentation necessary for blocks to fall under the list
+item, rather than having a fixed and arbitrary number.  The writer can
+think of the body of the list item as a unit which gets indented to the
+right enough to fit the list marker (and any indentation on the list
+marker).  (The laziness rule, #5, then allows continuation lines to be
+unindented if needed.)
+
+This rule is superior, we claim, to any rule requiring a fixed level of
+indentation from the margin.  The four-space rule is clear but
+unnatural. It is quite unintuitive that
+
+``` markdown
+- foo
+
+  bar
+
+  - baz
+```
+
+should be parsed as two lists with an intervening paragraph,
+
+``` html
+
      
+
    • foo
      • 
+
    
+
    bar
    
+
      
+
    • baz
      • 
+
    
+```
+
+as the four-space rule demands, rather than a single list,
+
+``` html
+
      
+
    • 
+
      foo
      
+
      bar
      
+
        
+
      • baz
        • 
+
      
+
      • 
+
    
+```
+
+The choice of four spaces is arbitrary.  It can be learned, but it is
+not likely to be guessed, and it trips up beginners regularly.
+
+Would it help to adopt a two-space rule?  The problem is that such
+a rule, together with the rule allowing up to three spaces of indentation for
+the initial list marker, allows text that is indented *less than* the
+original list marker to be included in the list item. For example,
+`Markdown.pl` parses
+
+``` markdown
+   - one
+
+  two
+```
+
+as a single list item, with `two` a continuation paragraph:
+
+``` html
+
      
+
    • 
+
      one
      
+
      two
      
+
      • 
+
    
+```
+
+and similarly
+
+``` markdown
+>   - one
+>
+>  two
+```
+
+as
+
+``` html
+
    
+
      
+
    • 
+
      one
      
+
      two
      
+
      • 
+
    
+
    
+```
+
+This is extremely unintuitive.
+
+Rather than requiring a fixed indent from the margin, we could require
+a fixed indent (say, two spaces, or even one space) from the list marker (which
+may itself be indented).  This proposal would remove the last anomaly
+discussed.  Unlike the spec presented above, it would count the following
+as a list item with a subparagraph, even though the paragraph `bar`
+is not indented as far as the first paragraph `foo`:
+
+``` markdown
+ 10. foo
+
+   bar  
+```
+
+Arguably this text does read like a list item with `bar` as a subparagraph,
+which may count in favor of the proposal.  However, on this proposal indented
+code would have to be indented six spaces after the list marker.  And this
+would break a lot of existing Markdown, which has the pattern:
+
+``` markdown
+1.  foo
+
+        indented code
+```
+
+where the code is indented eight spaces.  The spec above, by contrast, will
+parse this text as expected, since the code block's indentation is measured
+from the beginning of `foo`.
+
+The one case that needs special treatment is a list item that *starts*
+with indented code.  How much indentation is required in that case, since
+we don't have a "first paragraph" to measure from?  Rule #2 simply stipulates
+that in such cases, we require one space indentation from the list marker
+(and then the normal four spaces for the indented code).  This will match the
+four-space rule in cases where the list marker plus its initial indentation
+takes four spaces (a common case), but diverge in other cases.
+
+## Lists
+
+A [list](@) is a sequence of one or more
+list items [of the same type].  The list items
+may be separated by any number of blank lines.
+
+Two list items are [of the same type](@)
+if they begin with a [list marker] of the same type.
+Two list markers are of the
+same type if (a) they are bullet list markers using the same character
+(`-`, `+`, or `*`) or (b) they are ordered list numbers with the same
+delimiter (either `.` or `)`).
+
+A list is an [ordered list](@)
+if its constituent list items begin with
+[ordered list markers], and a
+[bullet list](@) if its constituent list
+items begin with [bullet list markers].
+
+The [start number](@)
+of an [ordered list] is determined by the list number of
+its initial list item.  The numbers of subsequent list items are
+disregarded.
+
+A list is [loose](@) if any of its constituent
+list items are separated by blank lines, or if any of its constituent
+list items directly contain two block-level elements with a blank line
+between them.  Otherwise a list is [tight](@).
+(The difference in HTML output is that paragraphs in a loose list are
+wrapped in `
    ` tags, while paragraphs in a tight list are not.)
+
+Changing the bullet or ordered list delimiter starts a new list:
+
+```````````````````````````````` example
+- foo
+- bar
++ baz
+.
+
      
+
    • foo
      • 
+
    • bar
      • 
+
    
+
      
+
    • baz
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+1. foo
+2. bar
+3) baz
+.
+
      
+
    1. foo
      2. 
+
    2. bar
      3. 
+
    
+
      
+
    1. baz
      2. 
+
    
+````````````````````````````````
+
+
+In CommonMark, a list can interrupt a paragraph. That is,
+no blank line is needed to separate a paragraph from a following
+list:
+
+```````````````````````````````` example
+Foo
+- bar
+- baz
+.
+
    Foo
    
+
      
+
    • bar
      • 
+
    • baz
      • 
+
    
+````````````````````````````````
+
+`Markdown.pl` does not allow this, through fear of triggering a list
+via a numeral in a hard-wrapped line:
+
+``` markdown
+The number of windows in my house is
+14.  The number of doors is 6.
+```
+
+Oddly, though, `Markdown.pl` *does* allow a blockquote to
+interrupt a paragraph, even though the same considerations might
+apply.
+
+In CommonMark, we do allow lists to interrupt paragraphs, for
+two reasons.  First, it is natural and not uncommon for people
+to start lists without blank lines:
+
+``` markdown
+I need to buy
+- new shoes
+- a coat
+- a plane ticket
+```
+
+Second, we are attracted to a
+
+> [principle of uniformity](@):
+> if a chunk of text has a certain
+> meaning, it will continue to have the same meaning when put into a
+> container block (such as a list item or blockquote).
+
+(Indeed, the spec for [list items] and [block quotes] presupposes
+this principle.) This principle implies that if
+
+``` markdown
+  * I need to buy
+    - new shoes
+    - a coat
+    - a plane ticket
+```
+
+is a list item containing a paragraph followed by a nested sublist,
+as all Markdown implementations agree it is (though the paragraph
+may be rendered without `
    ` tags, since the list is "tight"),
+then
+
+``` markdown
+I need to buy
+- new shoes
+- a coat
+- a plane ticket
+```
+
+by itself should be a paragraph followed by a nested sublist.
+
+Since it is well established Markdown practice to allow lists to
+interrupt paragraphs inside list items, the [principle of
+uniformity] requires us to allow this outside list items as
+well.  ([reStructuredText](https://docutils.sourceforge.net/rst.html)
+takes a different approach, requiring blank lines before lists
+even inside other list items.)
+
+In order to solve the problem of unwanted lists in paragraphs with
+hard-wrapped numerals, we allow only lists starting with `1` to
+interrupt paragraphs.  Thus,
+
+```````````````````````````````` example
+The number of windows in my house is
+14.  The number of doors is 6.
+.
+
    The number of windows in my house is
+14.  The number of doors is 6.
    
+````````````````````````````````
+
+We may still get an unintended result in cases like
+
+```````````````````````````````` example
+The number of windows in my house is
+1.  The number of doors is 6.
+.
+
    The number of windows in my house is
    
+
      
+
    1. The number of doors is 6.
      2. 
+
    
+````````````````````````````````
+
+but this rule should prevent most spurious list captures.
+
+There can be any number of blank lines between items:
+
+```````````````````````````````` example
+- foo
+
+- bar
+
+
+- baz
+.
+
      
+
    • 
+
      foo
      
+
      • 
+
    • 
+
      bar
      
+
      • 
+
    • 
+
      baz
      
+
      • 
+
    
+````````````````````````````````
+
+```````````````````````````````` example
+- foo
+  - bar
+    - baz
+
+
+      bim
+.
+
      
+
    • foo
+
        
+
      • bar
+
          
+
        • 
+
          baz
          
+
          bim
          
+
          • 
+
        
+
        • 
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+To separate consecutive lists of the same type, or to separate a
+list from an indented code block that would otherwise be parsed
+as a subparagraph of the final list item, you can insert a blank HTML
+comment:
+
+```````````````````````````````` example
+- foo
+- bar
+
+
+
+- baz
+- bim
+.
+
      
+
    • foo
      • 
+
    • bar
      • 
+
    
+
+
      
+
    • baz
      • 
+
    • bim
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+-   foo
+
+    notcode
+
+-   foo
+
+
+
+    code
+.
+
      
+
    • 
+
      foo
      
+
      notcode
      
+
      • 
+
    • 
+
      foo
      
+
      • 
+
    
+
+
    code
+
    
+````````````````````````````````
+
+
+List items need not be indented to the same level.  The following
+list items will be treated as items at the same list level,
+since none is indented enough to belong to the previous list
+item:
+
+```````````````````````````````` example
+- a
+ - b
+  - c
+   - d
+  - e
+ - f
+- g
+.
+
      
+
    • a
      • 
+
    • b
      • 
+
    • c
      • 
+
    • d
      • 
+
    • e
      • 
+
    • f
      • 
+
    • g
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+1. a
+
+  2. b
+
+   3. c
+.
+
      
+
    1. 
+
      a
      
+
      2. 
+
    2. 
+
      b
      
+
      3. 
+
    3. 
+
      c
      
+
      4. 
+
    
+````````````````````````````````
+
+Note, however, that list items must not be preceded by more than
+three spaces of indentation.  Here `- e` is treated as a paragraph continuation
+line, because it is indented more than three spaces:
+
+```````````````````````````````` example
+- a
+ - b
+  - c
+   - d
+    - e
+.
+
      
+
    • a
      • 
+
    • b
      • 
+
    • c
      • 
+
    • d
+- e
      • 
+
    
+````````````````````````````````
+
+And here, `3. c` is treated as in indented code block,
+because it is indented four spaces and preceded by a
+blank line.
+
+```````````````````````````````` example
+1. a
+
+  2. b
+
+    3. c
+.
+
      
+
    1. 
+
      a
      
+
      2. 
+
    2. 
+
      b
      
+
      3. 
+
    
+
    3. c
+
    
+````````````````````````````````
+
+
+This is a loose list, because there is a blank line between
+two of the list items:
+
+```````````````````````````````` example
+- a
+- b
+
+- c
+.
+
      
+
    • 
+
      a
      
+
      • 
+
    • 
+
      b
      
+
      • 
+
    • 
+
      c
      
+
      • 
+
    
+````````````````````````````````
+
+
+So is this, with a empty second item:
+
+```````````````````````````````` example
+* a
+*
+
+* c
+.
+
      
+
    • 
+
      a
      
+
      • 
+
    • 
+
    • 
+
      c
      
+
      • 
+
    
+````````````````````````````````
+
+
+These are loose lists, even though there are no blank lines between the items,
+because one of the items directly contains two block-level elements
+with a blank line between them:
+
+```````````````````````````````` example
+- a
+- b
+
+  c
+- d
+.
+
      
+
    • 
+
      a
      
+
      • 
+
    • 
+
      b
      
+
      c
      
+
      • 
+
    • 
+
      d
      
+
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+- b
+
+  [ref]: /url
+- d
+.
+
      
+
    • 
+
      a
      
+
      • 
+
    • 
+
      b
      
+
      • 
+
    • 
+
      d
      
+
      • 
+
    
+````````````````````````````````
+
+
+This is a tight list, because the blank lines are in a code block:
+
+```````````````````````````````` example
+- a
+- ```
+  b
+
+
+  ```
+- c
+.
+
      
+
    • a
      • 
+
    • 
+
      b
+
+
+
      
+
      • 
+
    • c
      • 
+
    
+````````````````````````````````
+
+
+This is a tight list, because the blank line is between two
+paragraphs of a sublist.  So the sublist is loose while
+the outer list is tight:
+
+```````````````````````````````` example
+- a
+  - b
+
+    c
+- d
+.
+
      
+
    • a
+
        
+
      • 
+
        b
        
+
        c
        
+
        • 
+
      
+
      • 
+
    • d
      • 
+
    
+````````````````````````````````
+
+
+This is a tight list, because the blank line is inside the
+block quote:
+
+```````````````````````````````` example
+* a
+  > b
+  >
+* c
+.
+
      
+
    • a
+
      
+
      b
      
+
      
+
      • 
+
    • c
      • 
+
    
+````````````````````````````````
+
+
+This list is tight, because the consecutive block elements
+are not separated by blank lines:
+
+```````````````````````````````` example
+- a
+  > b
+  ```
+  c
+  ```
+- d
+.
+
      
+
    • a
+
      
+
      b
      
+
      
+
      c
+
      
+
      • 
+
    • d
      • 
+
    
+````````````````````````````````
+
+
+A single-paragraph list is tight:
+
+```````````````````````````````` example
+- a
+.
+
      
+
    • a
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+  - b
+.
+
      
+
    • a
+
        
+
      • b
        • 
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+This list is loose, because of the blank line between the
+two block elements in the list item:
+
+```````````````````````````````` example
+1. ```
+   foo
+   ```
+
+   bar
+.
+
      
+
    1. 
+
      foo
+
      
+
      bar
      
+
      2. 
+
    
+````````````````````````````````
+
+
+Here the outer list is loose, the inner list tight:
+
+```````````````````````````````` example
+* foo
+  * bar
+
+  baz
+.
+
      
+
    • 
+
      foo
      
+
        
+
      • bar
        • 
+
      
+
      baz
      
+
      • 
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+  - b
+  - c
+
+- d
+  - e
+  - f
+.
+
      
+
    • 
+
      a
      
+
        
+
      • b
        • 
+
      • c
        • 
+
      
+
      • 
+
    • 
+
      d
      
+
        
+
      • e
        • 
+
      • f
        • 
+
      
+
      • 
+
    
+````````````````````````````````
+
+
+# Inlines
+
+Inlines are parsed sequentially from the beginning of the character
+stream to the end (left to right, in left-to-right languages).
+Thus, for example, in
+
+```````````````````````````````` example
+`hi`lo`
+.
+
    hilo`
    
+````````````````````````````````
+
+`hi` is parsed as code, leaving the backtick at the end as a literal
+backtick.
+
+
+
+## Code spans
+
+A [backtick string](@)
+is a string of one or more backtick characters (`` ` ``) that is neither
+preceded nor followed by a backtick.
+
+A [code span](@) begins with a backtick string and ends with
+a backtick string of equal length.  The contents of the code span are
+the characters between these two backtick strings, normalized in the
+following ways:
+
+- First, [line endings] are converted to [spaces].
+- If the resulting string both begins *and* ends with a [space]
+  character, but does not consist entirely of [space]
+  characters, a single [space] character is removed from the
+  front and back.  This allows you to include code that begins
+  or ends with backtick characters, which must be separated by
+  whitespace from the opening or closing backtick strings.
+
+This is a simple code span:
+
+```````````````````````````````` example
+`foo`
+.
+
    foo
    
+````````````````````````````````
+
+
+Here two backticks are used, because the code contains a backtick.
+This example also illustrates stripping of a single leading and
+trailing space:
+
+```````````````````````````````` example
+`` foo ` bar ``
+.
+
    foo ` bar
    
+````````````````````````````````
+
+
+This example shows the motivation for stripping leading and trailing
+spaces:
+
+```````````````````````````````` example
+` `` `
+.
+
    ``
    
+````````````````````````````````
+
+Note that only *one* space is stripped:
+
+```````````````````````````````` example
+`  ``  `
+.
+
     `` 
    
+````````````````````````````````
+
+The stripping only happens if the space is on both
+sides of the string:
+
+```````````````````````````````` example
+` a`
+.
+
     a
    
+````````````````````````````````
+
+Only [spaces], and not [unicode whitespace] in general, are
+stripped in this way:
+
+```````````````````````````````` example
+` b `
+.
+
     b 
    
+````````````````````````````````
+
+No stripping occurs if the code span contains only spaces:
+
+```````````````````````````````` example
+` `
+`  `
+.
+
     
+  
    
+````````````````````````````````
+
+
+[Line endings] are treated like spaces:
+
+```````````````````````````````` example
+``
+foo
+bar  
+baz
+``
+.
+
    foo bar   baz
    
+````````````````````````````````
+
+```````````````````````````````` example
+``
+foo 
+``
+.
+
    foo 
    
+````````````````````````````````
+
+
+Interior spaces are not collapsed:
+
+```````````````````````````````` example
+`foo   bar 
+baz`
+.
+
    foo   bar  baz
    
+````````````````````````````````
+
+Note that browsers will typically collapse consecutive spaces
+when rendering `` elements, so it is recommended that
+the following CSS be used:
+
+    code{white-space: pre-wrap;}
+
+
+Note that backslash escapes do not work in code spans. All backslashes
+are treated literally:
+
+```````````````````````````````` example
+`foo\`bar`
+.
+
    foo\bar`
    
+````````````````````````````````
+
+
+Backslash escapes are never needed, because one can always choose a
+string of *n* backtick characters as delimiters, where the code does
+not contain any strings of exactly *n* backtick characters.
+
+```````````````````````````````` example
+``foo`bar``
+.
+
    foo`bar
    
+````````````````````````````````
+
+```````````````````````````````` example
+` foo `` bar `
+.
+
    foo `` bar
    
+````````````````````````````````
+
+
+Code span backticks have higher precedence than any other inline
+constructs except HTML tags and autolinks.  Thus, for example, this is
+not parsed as emphasized text, since the second `*` is part of a code
+span:
+
+```````````````````````````````` example
+*foo`*`
+.
+
    *foo*
    
+````````````````````````````````
+
+
+And this is not parsed as a link:
+
+```````````````````````````````` example
+[not a `link](/foo`)
+.
+
    [not a link](/foo)
    
+````````````````````````````````
+
+
+Code spans, HTML tags, and autolinks have the same precedence.
+Thus, this is code:
+
+```````````````````````````````` example
+``
+.
+
    <a href="">`
    
+````````````````````````````````
+
+
+But this is an HTML tag:
+
+```````````````````````````````` example
+    `
+.
+
    `
    
+````````````````````````````````
+
+
+And this is code:
+
+```````````````````````````````` example
+``
+.
+
    <https://foo.bar.baz>`
    
+````````````````````````````````
+
+
+But this is an autolink:
+
+```````````````````````````````` example
+`
+.
+
    https://foo.bar.`baz`
    
+````````````````````````````````
+
+
+When a backtick string is not closed by a matching backtick string,
+we just have literal backticks:
+
+```````````````````````````````` example
+```foo``
+.
+
    ```foo``
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+`foo
+.
+
    `foo
    
+````````````````````````````````
+
+The following case also illustrates the need for opening and
+closing backtick strings to be equal in length:
+
+```````````````````````````````` example
+`foo``bar``
+.
+
    `foobar
    
+````````````````````````````````
+
+
+## Emphasis and strong emphasis
+
+John Gruber's original [Markdown syntax
+description](https://daringfireball.net/projects/markdown/syntax#em) says:
+
+> Markdown treats asterisks (`*`) and underscores (`_`) as indicators of
+> emphasis. Text wrapped with one `*` or `_` will be wrapped with an HTML
+> `` tag; double `*`'s or `_`'s will be wrapped with an HTML ``
+> tag.
+
+This is enough for most users, but these rules leave much undecided,
+especially when it comes to nested emphasis.  The original
+`Markdown.pl` test suite makes it clear that triple `***` and
+`___` delimiters can be used for strong emphasis, and most
+implementations have also allowed the following patterns:
+
+``` markdown
+***strong emph***
+***strong** in emph*
+***emph* in strong**
+**in strong *emph***
+*in emph **strong***
+```
+
+The following patterns are less widely supported, but the intent
+is clear and they are useful (especially in contexts like bibliography
+entries):
+
+``` markdown
+*emph *with emph* in it*
+**strong **with strong** in it**
+```
+
+Many implementations have also restricted intraword emphasis to
+the `*` forms, to avoid unwanted emphasis in words containing
+internal underscores.  (It is best practice to put these in code
+spans, but users often do not.)
+
+``` markdown
+internal emphasis: foo*bar*baz
+no emphasis: foo_bar_baz
+```
+
+The rules given below capture all of these patterns, while allowing
+for efficient parsing strategies that do not backtrack.
+
+First, some definitions.  A [delimiter run](@) is either
+a sequence of one or more `*` characters that is not preceded or
+followed by a non-backslash-escaped `*` character, or a sequence
+of one or more `_` characters that is not preceded or followed by
+a non-backslash-escaped `_` character.
+
+A [left-flanking delimiter run](@) is
+a [delimiter run] that is (1) not followed by [Unicode whitespace],
+and either (2a) not followed by a [Unicode punctuation character], or
+(2b) followed by a [Unicode punctuation character] and
+preceded by [Unicode whitespace] or a [Unicode punctuation character].
+For purposes of this definition, the beginning and the end of
+the line count as Unicode whitespace.
+
+A [right-flanking delimiter run](@) is
+a [delimiter run] that is (1) not preceded by [Unicode whitespace],
+and either (2a) not preceded by a [Unicode punctuation character], or
+(2b) preceded by a [Unicode punctuation character] and
+followed by [Unicode whitespace] or a [Unicode punctuation character].
+For purposes of this definition, the beginning and the end of
+the line count as Unicode whitespace.
+
+Here are some examples of delimiter runs.
+
+  - left-flanking but not right-flanking:
+
+    ```
+    ***abc
+      _abc
+    **"abc"
+     _"abc"
+    ```
+
+  - right-flanking but not left-flanking:
+
+    ```
+     abc***
+     abc_
+    "abc"**
+    "abc"_
+    ```
+
+  - Both left and right-flanking:
+
+    ```
+     abc***def
+    "abc"_"def"
+    ```
+
+  - Neither left nor right-flanking:
+
+    ```
+    abc *** def
+    a _ b
+    ```
+
+(The idea of distinguishing left-flanking and right-flanking
+delimiter runs based on the character before and the character
+after comes from Roopesh Chander's
+[vfmd](https://web.archive.org/web/20220608143320/http://www.vfmd.org/vfmd-spec/specification/#procedure-for-identifying-emphasis-tags).
+vfmd uses the terminology "emphasis indicator string" instead of "delimiter
+run," and its rules for distinguishing left- and right-flanking runs
+are a bit more complex than the ones given here.)
+
+The following rules define emphasis and strong emphasis:
+
+1.  A single `*` character [can open emphasis](@)
+    iff (if and only if) it is part of a [left-flanking delimiter run].
+
+2.  A single `_` character [can open emphasis] iff
+    it is part of a [left-flanking delimiter run]
+    and either (a) not part of a [right-flanking delimiter run]
+    or (b) part of a [right-flanking delimiter run]
+    preceded by a [Unicode punctuation character].
+
+3.  A single `*` character [can close emphasis](@)
+    iff it is part of a [right-flanking delimiter run].
+
+4.  A single `_` character [can close emphasis] iff
+    it is part of a [right-flanking delimiter run]
+    and either (a) not part of a [left-flanking delimiter run]
+    or (b) part of a [left-flanking delimiter run]
+    followed by a [Unicode punctuation character].
+
+5.  A double `**` [can open strong emphasis](@)
+    iff it is part of a [left-flanking delimiter run].
+
+6.  A double `__` [can open strong emphasis] iff
+    it is part of a [left-flanking delimiter run]
+    and either (a) not part of a [right-flanking delimiter run]
+    or (b) part of a [right-flanking delimiter run]
+    preceded by a [Unicode punctuation character].
+
+7.  A double `**` [can close strong emphasis](@)
+    iff it is part of a [right-flanking delimiter run].
+
+8.  A double `__` [can close strong emphasis] iff
+    it is part of a [right-flanking delimiter run]
+    and either (a) not part of a [left-flanking delimiter run]
+    or (b) part of a [left-flanking delimiter run]
+    followed by a [Unicode punctuation character].
+
+9.  Emphasis begins with a delimiter that [can open emphasis] and ends
+    with a delimiter that [can close emphasis], and that uses the same
+    character (`_` or `*`) as the opening delimiter.  The
+    opening and closing delimiters must belong to separate
+    [delimiter runs].  If one of the delimiters can both
+    open and close emphasis, then the sum of the lengths of the
+    delimiter runs containing the opening and closing delimiters
+    must not be a multiple of 3 unless both lengths are
+    multiples of 3.
+
+10. Strong emphasis begins with a delimiter that
+    [can open strong emphasis] and ends with a delimiter that
+    [can close strong emphasis], and that uses the same character
+    (`_` or `*`) as the opening delimiter.  The
+    opening and closing delimiters must belong to separate
+    [delimiter runs].  If one of the delimiters can both open
+    and close strong emphasis, then the sum of the lengths of
+    the delimiter runs containing the opening and closing
+    delimiters must not be a multiple of 3 unless both lengths
+    are multiples of 3.
+
+11. A literal `*` character cannot occur at the beginning or end of
+    `*`-delimited emphasis or `**`-delimited strong emphasis, unless it
+    is backslash-escaped.
+
+12. A literal `_` character cannot occur at the beginning or end of
+    `_`-delimited emphasis or `__`-delimited strong emphasis, unless it
+    is backslash-escaped.
+
+Where rules 1--12 above are compatible with multiple parsings,
+the following principles resolve ambiguity:
+
+13. The number of nestings should be minimized. Thus, for example,
+    an interpretation `...` is always preferred to
+    `...`.
+
+14. An interpretation `...` is always
+    preferred to `...`.
+
+15. When two potential emphasis or strong emphasis spans overlap,
+    so that the second begins before the first ends and ends after
+    the first ends, the first takes precedence. Thus, for example,
+    `*foo _bar* baz_` is parsed as `foo _bar baz_` rather
+    than `*foo bar* baz`.
+
+16. When there are two potential emphasis or strong emphasis spans
+    with the same closing delimiter, the shorter one (the one that
+    opens later) takes precedence. Thus, for example,
+    `**foo **bar baz**` is parsed as `**foo bar baz`
+    rather than `foo **bar baz`.
+
+17. Inline code spans, links, images, and HTML tags group more tightly
+    than emphasis.  So, when there is a choice between an interpretation
+    that contains one of these elements and one that does not, the
+    former always wins.  Thus, for example, `*[foo*](bar)` is
+    parsed as `*foo*` rather than as
+    `[foo](bar)`.
+
+These rules can be illustrated through a series of examples.
+
+Rule 1:
+
+```````````````````````````````` example
+*foo bar*
+.
+
    foo bar
    
+````````````````````````````````
+
+
+This is not emphasis, because the opening `*` is followed by
+whitespace, and hence not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a * foo bar*
+.
+
    a * foo bar*
    
+````````````````````````````````
+
+
+This is not emphasis, because the opening `*` is preceded
+by an alphanumeric and followed by punctuation, and hence
+not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a*"foo"*
+.
+
    a*"foo"*
    
+````````````````````````````````
+
+
+Unicode nonbreaking spaces count as whitespace, too:
+
+```````````````````````````````` example
+* a *
+.
+
    * a *
    
+````````````````````````````````
+
+
+Unicode symbols count as punctuation, too:
+
+```````````````````````````````` example
+*$*alpha.
+
+*£*bravo.
+
+*€*charlie.
+
+*𞋿*delta.
+.
+
    *$*alpha.
    
+
    *£*bravo.
    
+
    *€*charlie.
    
+
    *𞋿*delta.
    
+````````````````````````````````
+
+
+Intraword emphasis with `*` is permitted:
+
+```````````````````````````````` example
+foo*bar*
+.
+
    foobar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+5*6*78
+.
+
    5678
    
+````````````````````````````````
+
+
+Rule 2:
+
+```````````````````````````````` example
+_foo bar_
+.
+
    foo bar
    
+````````````````````````````````
+
+
+This is not emphasis, because the opening `_` is followed by
+whitespace:
+
+```````````````````````````````` example
+_ foo bar_
+.
+
    _ foo bar_
    
+````````````````````````````````
+
+
+This is not emphasis, because the opening `_` is preceded
+by an alphanumeric and followed by punctuation:
+
+```````````````````````````````` example
+a_"foo"_
+.
+
    a_"foo"_
    
+````````````````````````````````
+
+
+Emphasis with `_` is not allowed inside words:
+
+```````````````````````````````` example
+foo_bar_
+.
+
    foo_bar_
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+5_6_78
+.
+
    5_6_78
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+пристаням_стремятся_
+.
+
    пристаням_стремятся_
    
+````````````````````````````````
+
+
+Here `_` does not generate emphasis, because the first delimiter run
+is right-flanking and the second left-flanking:
+
+```````````````````````````````` example
+aa_"bb"_cc
+.
+
    aa_"bb"_cc
    
+````````````````````````````````
+
+
+This is emphasis, even though the opening delimiter is
+both left- and right-flanking, because it is preceded by
+punctuation:
+
+```````````````````````````````` example
+foo-_(bar)_
+.
+
    foo-(bar)
    
+````````````````````````````````
+
+
+Rule 3:
+
+This is not emphasis, because the closing delimiter does
+not match the opening delimiter:
+
+```````````````````````````````` example
+_foo*
+.
+
    _foo*
    
+````````````````````````````````
+
+
+This is not emphasis, because the closing `*` is preceded by
+whitespace:
+
+```````````````````````````````` example
+*foo bar *
+.
+
    *foo bar *
    
+````````````````````````````````
+
+
+A line ending also counts as whitespace:
+
+```````````````````````````````` example
+*foo bar
+*
+.
+
    *foo bar
+*
    
+````````````````````````````````
+
+
+This is not emphasis, because the second `*` is
+preceded by punctuation and followed by an alphanumeric
+(hence it is not part of a [right-flanking delimiter run]:
+
+```````````````````````````````` example
+*(*foo)
+.
+
    *(*foo)
    
+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with this example:
+
+```````````````````````````````` example
+*(*foo*)*
+.
+
    (foo)
    
+````````````````````````````````
+
+
+Intraword emphasis with `*` is allowed:
+
+```````````````````````````````` example
+*foo*bar
+.
+
    foobar
    
+````````````````````````````````
+
+
+
+Rule 4:
+
+This is not emphasis, because the closing `_` is preceded by
+whitespace:
+
+```````````````````````````````` example
+_foo bar _
+.
+
    _foo bar _
    
+````````````````````````````````
+
+
+This is not emphasis, because the second `_` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+_(_foo)
+.
+
    _(_foo)
    
+````````````````````````````````
+
+
+This is emphasis within emphasis:
+
+```````````````````````````````` example
+_(_foo_)_
+.
+
    (foo)
    
+````````````````````````````````
+
+
+Intraword emphasis is disallowed for `_`:
+
+```````````````````````````````` example
+_foo_bar
+.
+
    _foo_bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_пристаням_стремятся
+.
+
    _пристаням_стремятся
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo_bar_baz_
+.
+
    foo_bar_baz
    
+````````````````````````````````
+
+
+This is emphasis, even though the closing delimiter is
+both left- and right-flanking, because it is followed by
+punctuation:
+
+```````````````````````````````` example
+_(bar)_.
+.
+
    (bar).
    
+````````````````````````````````
+
+
+Rule 5:
+
+```````````````````````````````` example
+**foo bar**
+.
+
    foo bar
    
+````````````````````````````````
+
+
+This is not strong emphasis, because the opening delimiter is
+followed by whitespace:
+
+```````````````````````````````` example
+** foo bar**
+.
+
    ** foo bar**
    
+````````````````````````````````
+
+
+This is not strong emphasis, because the opening `**` is preceded
+by an alphanumeric and followed by punctuation, and hence
+not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a**"foo"**
+.
+
    a**"foo"**
    
+````````````````````````````````
+
+
+Intraword strong emphasis with `**` is permitted:
+
+```````````````````````````````` example
+foo**bar**
+.
+
    foobar
    
+````````````````````````````````
+
+
+Rule 6:
+
+```````````````````````````````` example
+__foo bar__
+.
+
    foo bar
    
+````````````````````````````````
+
+
+This is not strong emphasis, because the opening delimiter is
+followed by whitespace:
+
+```````````````````````````````` example
+__ foo bar__
+.
+
    __ foo bar__
    
+````````````````````````````````
+
+
+A line ending counts as whitespace:
+```````````````````````````````` example
+__
+foo bar__
+.
+
    __
+foo bar__
    
+````````````````````````````````
+
+
+This is not strong emphasis, because the opening `__` is preceded
+by an alphanumeric and followed by punctuation:
+
+```````````````````````````````` example
+a__"foo"__
+.
+
    a__"foo"__
    
+````````````````````````````````
+
+
+Intraword strong emphasis is forbidden with `__`:
+
+```````````````````````````````` example
+foo__bar__
+.
+
    foo__bar__
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+5__6__78
+.
+
    5__6__78
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+пристаням__стремятся__
+.
+
    пристаням__стремятся__
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo, __bar__, baz__
+.
+
    foo, bar, baz
    
+````````````````````````````````
+
+
+This is strong emphasis, even though the opening delimiter is
+both left- and right-flanking, because it is preceded by
+punctuation:
+
+```````````````````````````````` example
+foo-__(bar)__
+.
+
    foo-(bar)
    
+````````````````````````````````
+
+
+
+Rule 7:
+
+This is not strong emphasis, because the closing delimiter is preceded
+by whitespace:
+
+```````````````````````````````` example
+**foo bar **
+.
+
    **foo bar **
    
+````````````````````````````````
+
+
+(Nor can it be interpreted as an emphasized `*foo bar *`, because of
+Rule 11.)
+
+This is not strong emphasis, because the second `**` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+**(**foo)
+.
+
    **(**foo)
    
+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with these examples:
+
+```````````````````````````````` example
+*(**foo**)*
+.
+
    (foo)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**Gomphocarpus (*Gomphocarpus physocarpus*, syn.
+*Asclepias physocarpa*)**
+.
+
    Gomphocarpus (Gomphocarpus physocarpus, syn.
+Asclepias physocarpa)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo "*bar*" foo**
+.
+
    foo "bar" foo
    
+````````````````````````````````
+
+
+Intraword emphasis:
+
+```````````````````````````````` example
+**foo**bar
+.
+
    foobar
    
+````````````````````````````````
+
+
+Rule 8:
+
+This is not strong emphasis, because the closing delimiter is
+preceded by whitespace:
+
+```````````````````````````````` example
+__foo bar __
+.
+
    __foo bar __
    
+````````````````````````````````
+
+
+This is not strong emphasis, because the second `__` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+__(__foo)
+.
+
    __(__foo)
    
+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with this example:
+
+```````````````````````````````` example
+_(__foo__)_
+.
+
    (foo)
    
+````````````````````````````````
+
+
+Intraword strong emphasis is forbidden with `__`:
+
+```````````````````````````````` example
+__foo__bar
+.
+
    __foo__bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__пристаням__стремятся
+.
+
    __пристаням__стремятся
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo__bar__baz__
+.
+
    foo__bar__baz
    
+````````````````````````````````
+
+
+This is strong emphasis, even though the closing delimiter is
+both left- and right-flanking, because it is followed by
+punctuation:
+
+```````````````````````````````` example
+__(bar)__.
+.
+
    (bar).
    
+````````````````````````````````
+
+
+Rule 9:
+
+Any nonempty sequence of inline elements can be the contents of an
+emphasized span.
+
+```````````````````````````````` example
+*foo [bar](/url)*
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo
+bar*
+.
+
    foo
+bar
    
+````````````````````````````````
+
+
+In particular, emphasis and strong emphasis can be nested
+inside emphasis:
+
+```````````````````````````````` example
+_foo __bar__ baz_
+.
+
    foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo _bar_ baz_
+.
+
    foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo_ bar_
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo *bar**
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo **bar** baz*
+.
+
    foo bar baz
    
+````````````````````````````````
+
+```````````````````````````````` example
+*foo**bar**baz*
+.
+
    foobarbaz
    
+````````````````````````````````
+
+Note that in the preceding case, the interpretation
+
+``` markdown
+
    foobarbaz
    
+```
+
+
+is precluded by the condition that a delimiter that
+can both open and close (like the `*` after `foo`)
+cannot form emphasis if the sum of the lengths of
+the delimiter runs containing the opening and
+closing delimiters is a multiple of 3 unless
+both lengths are multiples of 3.
+
+
+For the same reason, we don't get two consecutive
+emphasis sections in this example:
+
+```````````````````````````````` example
+*foo**bar*
+.
+
    foo**bar
    
+````````````````````````````````
+
+
+The same condition ensures that the following
+cases are all strong emphasis nested inside
+emphasis, even when the interior whitespace is
+omitted:
+
+
+```````````````````````````````` example
+***foo** bar*
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo **bar***
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo**bar***
+.
+
    foobar
    
+````````````````````````````````
+
+
+When the lengths of the interior closing and opening
+delimiter runs are *both* multiples of 3, though,
+they can match to create emphasis:
+
+```````````````````````````````` example
+foo***bar***baz
+.
+
    foobarbaz
    
+````````````````````````````````
+
+```````````````````````````````` example
+foo******bar*********baz
+.
+
    foobar***baz
    
+````````````````````````````````
+
+
+Indefinite levels of nesting are possible:
+
+```````````````````````````````` example
+*foo **bar *baz* bim** bop*
+.
+
    foo bar baz bim bop
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo [*bar*](/url)*
+.
+
    foo bar
    
+````````````````````````````````
+
+
+There can be no empty emphasis or strong emphasis:
+
+```````````````````````````````` example
+** is not an empty emphasis
+.
+
    ** is not an empty emphasis
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**** is not an empty strong emphasis
+.
+
    **** is not an empty strong emphasis
    
+````````````````````````````````
+
+
+
+Rule 10:
+
+Any nonempty sequence of inline elements can be the contents of an
+strongly emphasized span.
+
+```````````````````````````````` example
+**foo [bar](/url)**
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo
+bar**
+.
+
    foo
+bar
    
+````````````````````````````````
+
+
+In particular, emphasis and strong emphasis can be nested
+inside strong emphasis:
+
+```````````````````````````````` example
+__foo _bar_ baz__
+.
+
    foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo __bar__ baz__
+.
+
    foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo__ bar__
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo **bar****
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo *bar* baz**
+.
+
    foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo*bar*baz**
+.
+
    foobarbaz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+***foo* bar**
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo *bar***
+.
+
    foo bar
    
+````````````````````````````````
+
+
+Indefinite levels of nesting are possible:
+
+```````````````````````````````` example
+**foo *bar **baz**
+bim* bop**
+.
+
    foo bar baz
+bim bop
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo [*bar*](/url)**
+.
+
    foo bar
    
+````````````````````````````````
+
+
+There can be no empty emphasis or strong emphasis:
+
+```````````````````````````````` example
+__ is not an empty emphasis
+.
+
    __ is not an empty emphasis
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+____ is not an empty strong emphasis
+.
+
    ____ is not an empty strong emphasis
    
+````````````````````````````````
+
+
+
+Rule 11:
+
+```````````````````````````````` example
+foo ***
+.
+
    foo ***
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *\**
+.
+
    foo *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *_*
+.
+
    foo _
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *****
+.
+
    foo *****
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo **\***
+.
+
    foo *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo **_**
+.
+
    foo _
    
+````````````````````````````````
+
+
+Note that when delimiters do not match evenly, Rule 11 determines
+that the excess literal `*` characters will appear outside of the
+emphasis, rather than inside it:
+
+```````````````````````````````` example
+**foo*
+.
+
    *foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo**
+.
+
    foo*
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+***foo**
+.
+
    *foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+****foo*
+.
+
    ***foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo***
+.
+
    foo*
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo****
+.
+
    foo***
    
+````````````````````````````````
+
+
+
+Rule 12:
+
+```````````````````````````````` example
+foo ___
+.
+
    foo ___
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _\__
+.
+
    foo _
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _*_
+.
+
    foo *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _____
+.
+
    foo _____
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo __\___
+.
+
    foo _
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo __*__
+.
+
    foo *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo_
+.
+
    _foo
    
+````````````````````````````````
+
+
+Note that when delimiters do not match evenly, Rule 12 determines
+that the excess literal `_` characters will appear outside of the
+emphasis, rather than inside it:
+
+```````````````````````````````` example
+_foo__
+.
+
    foo_
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+___foo__
+.
+
    _foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo_
+.
+
    ___foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo___
+.
+
    foo_
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo____
+.
+
    foo___
    
+````````````````````````````````
+
+
+Rule 13 implies that if you want emphasis nested directly inside
+emphasis, you must use different delimiters:
+
+```````````````````````````````` example
+**foo**
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*_foo_*
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo__
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_*foo*_
+.
+
    foo
    
+````````````````````````````````
+
+
+However, strong emphasis within strong emphasis is possible without
+switching delimiters:
+
+```````````````````````````````` example
+****foo****
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo____
+.
+
    foo
    
+````````````````````````````````
+
+
+
+Rule 13 can be applied to arbitrarily long sequences of
+delimiters:
+
+```````````````````````````````` example
+******foo******
+.
+
    foo
    
+````````````````````````````````
+
+
+Rule 14:
+
+```````````````````````````````` example
+***foo***
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_____foo_____
+.
+
    foo
    
+````````````````````````````````
+
+
+Rule 15:
+
+```````````````````````````````` example
+*foo _bar* baz_
+.
+
    foo _bar baz_
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo __bar *baz bim__ bam*
+.
+
    foo bar *baz bim bam
    
+````````````````````````````````
+
+
+Rule 16:
+
+```````````````````````````````` example
+**foo **bar baz**
+.
+
    **foo bar baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo *bar baz*
+.
+
    *foo bar baz
    
+````````````````````````````````
+
+
+Rule 17:
+
+```````````````````````````````` example
+*[bar*](/url)
+.
+
    *bar*
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo [bar_](/url)
+.
+
    _foo bar_
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*
+.
+
    *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**
+.
+
    **
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__
+.
+
    __
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*a `*`*
+.
+
    a *
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+_a `_`_
+.
+
    a _
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+**a
+.
+
    **ahttps://foo.bar/?q=**
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+__a
+.
+
    __ahttps://foo.bar/?q=__
    
+````````````````````````````````
+
+
+
+## Links
+
+A link contains [link text] (the visible text), a [link destination]
+(the URI that is the link destination), and optionally a [link title].
+There are two basic kinds of links in Markdown.  In [inline links] the
+destination and title are given immediately after the link text.  In
+[reference links] the destination and title are defined elsewhere in
+the document.
+
+A [link text](@) consists of a sequence of zero or more
+inline elements enclosed by square brackets (`[` and `]`).  The
+following rules apply:
+
+- Links cannot contain other links, at any level of nesting. If
+  multiple otherwise valid link definitions appear nested inside each
+  other, the inner-most definition is used.
+
+- Brackets are allowed in the [link text] only if (a) they
+  are backslash-escaped or (b) they appear as a matched pair of brackets,
+  with an open bracket `[`, a sequence of zero or more inlines, and
+  a close bracket `]`.
+
+- Backtick [code spans], [autolinks], and raw [HTML tags] bind more tightly
+  than the brackets in link text.  Thus, for example,
+  `` [foo`]` `` could not be a link text, since the second `]`
+  is part of a code span.
+
+- The brackets in link text bind more tightly than markers for
+  [emphasis and strong emphasis]. Thus, for example, `*[foo*](url)` is a link.
+
+A [link destination](@) consists of either
+
+- a sequence of zero or more characters between an opening `<` and a
+  closing `>` that contains no line endings or unescaped
+  `<` or `>` characters, or
+
+- a nonempty sequence of characters that does not start with `<`,
+  does not include [ASCII control characters][ASCII control character]
+  or [space] character, and includes parentheses only if (a) they are
+  backslash-escaped or (b) they are part of a balanced pair of
+  unescaped parentheses.
+  (Implementations may impose limits on parentheses nesting to
+  avoid performance issues, but at least three levels of nesting
+  should be supported.)
+
+A [link title](@)  consists of either
+
+- a sequence of zero or more characters between straight double-quote
+  characters (`"`), including a `"` character only if it is
+  backslash-escaped, or
+
+- a sequence of zero or more characters between straight single-quote
+  characters (`'`), including a `'` character only if it is
+  backslash-escaped, or
+
+- a sequence of zero or more characters between matching parentheses
+  (`(...)`), including a `(` or `)` character only if it is
+  backslash-escaped.
+
+Although [link titles] may span multiple lines, they must not contain
+a [blank line].
+
+An [inline link](@) consists of a [link text] followed immediately
+by a left parenthesis `(`, an optional [link destination], an optional
+[link title], and a right parenthesis `)`.
+These four components may be separated by spaces, tabs, and up to one line
+ending.
+If both [link destination] and [link title] are present, they *must* be
+separated by spaces, tabs, and up to one line ending.
+
+The link's text consists of the inlines contained
+in the [link text] (excluding the enclosing square brackets).
+The link's URI consists of the link destination, excluding enclosing
+`<...>` if present, with backslash-escapes in effect as described
+above.  The link's title consists of the link title, excluding its
+enclosing delimiters, with backslash-escapes in effect as described
+above.
+
+Here is a simple inline link:
+
+```````````````````````````````` example
+[link](/uri "title")
+.
+
    link
    
+````````````````````````````````
+
+
+The title, the link text and even 
+the destination may be omitted:
+
+```````````````````````````````` example
+[link](/uri)
+.
+
    link
    
+````````````````````````````````
+
+```````````````````````````````` example
+[](./target.md)
+.
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link]()
+.
+
    link
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link](<>)
+.
+
    link
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[]()
+.
+
    
+````````````````````````````````
+
+The destination can only contain spaces if it is
+enclosed in angle brackets:
+
+```````````````````````````````` example
+[link](/my uri)
+.
+
    [link](/my uri)
    
+````````````````````````````````
+
+```````````````````````````````` example
+[link](    )
+.
+
    link
    
+````````````````````````````````
+
+The destination cannot contain line endings,
+even if enclosed in angle brackets:
+
+```````````````````````````````` example
+[link](foo
+bar)
+.
+
    [link](foo
+bar)
    
+````````````````````````````````
+
+```````````````````````````````` example
+[link]()
+.
+
    [link]()
    
+````````````````````````````````
+
+The destination can contain `)` if it is enclosed
+in angle brackets:
+
+```````````````````````````````` example
+[a]()
+.
+
    a
    
+````````````````````````````````
+
+Angle brackets that enclose links must be unescaped:
+
+```````````````````````````````` example
+[link]()
+.
+
    [link](<foo>)
    
+````````````````````````````````
+
+These are not links, because the opening angle bracket
+is not matched properly:
+
+```````````````````````````````` example
+[a](
+[a](c)
+.
+
    [a](<b)c
+[a](<b)c>
+[a](c)
    
+````````````````````````````````
+
+Parentheses inside the link destination may be escaped:
+
+```````````````````````````````` example
+[link](\(foo\))
+.
+
    link
    
+````````````````````````````````
+
+Any number of parentheses are allowed without escaping, as long as they are
+balanced:
+
+```````````````````````````````` example
+[link](foo(and(bar)))
+.
+
    link
    
+````````````````````````````````
+
+However, if you have unbalanced parentheses, you need to escape or use the
+`<...>` form:
+
+```````````````````````````````` example
+[link](foo(and(bar))
+.
+
    [link](foo(and(bar))
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link](foo\(and\(bar\))
+.
+
    link
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link]()
+.
+
    link
    
+````````````````````````````````
+
+
+Parentheses and other symbols can also be escaped, as usual
+in Markdown:
+
+```````````````````````````````` example
+[link](foo\)\:)
+.
+
    link
    
+````````````````````````````````
+
+
+A link can contain fragment identifiers and queries:
+
+```````````````````````````````` example
+[link](#fragment)
+
+[link](https://example.com#fragment)
+
+[link](https://example.com?foo=3#frag)
+.
+
    link
    
+
    link
    
+
    link
    
+````````````````````````````````
+
+
+Note that a backslash before a non-escapable character is
+just a backslash:
+
+```````````````````````````````` example
+[link](foo\bar)
+.
+
    link
    
+````````````````````````````````
+
+
+URL-escaping should be left alone inside the destination, as all
+URL-escaped characters are also valid URL characters. Entity and
+numerical character references in the destination will be parsed
+into the corresponding Unicode code points, as usual.  These may
+be optionally URL-escaped when written as HTML, but this spec
+does not enforce any particular policy for rendering URLs in
+HTML or other formats.  Renderers may make different decisions
+about how to escape or normalize URLs in the output.
+
+```````````````````````````````` example
+[link](foo%20bä)
+.
+
    link
    
+````````````````````````````````
+
+
+Note that, because titles can often be parsed as destinations,
+if you try to omit the destination and keep the title, you'll
+get unexpected results:
+
+```````````````````````````````` example
+[link]("title")
+.
+
    link
    
+````````````````````````````````
+
+
+Titles may be in single quotes, double quotes, or parentheses:
+
+```````````````````````````````` example
+[link](/url "title")
+[link](/url 'title')
+[link](/url (title))
+.
+
    link
+link
+link
    
+````````````````````````````````
+
+
+Backslash escapes and entity and numeric character references
+may be used in titles:
+
+```````````````````````````````` example
+[link](/url "title \""")
+.
+
    link
    
+````````````````````````````````
+
+
+Titles must be separated from the link using spaces, tabs, and up to one line
+ending.
+Other [Unicode whitespace] like non-breaking space doesn't work.
+
+```````````````````````````````` example
+[link](/url "title")
+.
+
    link
    
+````````````````````````````````
+
+
+Nested balanced quotes are not allowed without escaping:
+
+```````````````````````````````` example
+[link](/url "title "and" title")
+.
+
    [link](/url "title "and" title")
    
+````````````````````````````````
+
+
+But it is easy to work around this by using a different quote type:
+
+```````````````````````````````` example
+[link](/url 'title "and" title')
+.
+
    link
    
+````````````````````````````````
+
+
+(Note:  `Markdown.pl` did allow double quotes inside a double-quoted
+title, and its test suite included a test demonstrating this.
+But it is hard to see a good rationale for the extra complexity this
+brings, since there are already many ways---backslash escaping,
+entity and numeric character references, or using a different
+quote type for the enclosing title---to write titles containing
+double quotes.  `Markdown.pl`'s handling of titles has a number
+of other strange features.  For example, it allows single-quoted
+titles in inline links, but not reference links.  And, in
+reference links but not inline links, it allows a title to begin
+with `"` and end with `)`.  `Markdown.pl` 1.0.1 even allows
+titles with no closing quotation mark, though 1.0.2b8 does not.
+It seems preferable to adopt a simple, rational rule that works
+the same way in inline links and link reference definitions.)
+
+Spaces, tabs, and up to one line ending is allowed around the destination and
+title:
+
+```````````````````````````````` example
+[link](   /uri
+  "title"  )
+.
+
    link
    
+````````````````````````````````
+
+
+But it is not allowed between the link text and the
+following parenthesis:
+
+```````````````````````````````` example
+[link] (/uri)
+.
+
    [link] (/uri)
    
+````````````````````````````````
+
+
+The link text may contain balanced brackets, but not unbalanced ones,
+unless they are escaped:
+
+```````````````````````````````` example
+[link [foo [bar]]](/uri)
+.
+
    link [foo [bar]]
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link] bar](/uri)
+.
+
    [link] bar](/uri)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link [bar](/uri)
+.
+
    [link bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link \[bar](/uri)
+.
+
    link [bar
    
+````````````````````````````````
+
+
+The link text may contain inline content:
+
+```````````````````````````````` example
+[link *foo **bar** `#`*](/uri)
+.
+
    link foo bar #
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[![moon](moon.jpg)](/uri)
+.
+
    moon
    
+````````````````````````````````
+
+
+However, links cannot contain other links, at any level of nesting.
+
+```````````````````````````````` example
+[foo [bar](/uri)](/uri)
+.
+
    [foo bar](/uri)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *[bar [baz](/uri)](/uri)*](/uri)
+.
+
    [foo [bar baz](/uri)](/uri)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![[[foo](uri1)](uri2)](uri3)
+.
+
    [foo](uri2)
    
+````````````````````````````````
+
+
+These cases illustrate the precedence of link text grouping over
+emphasis grouping:
+
+```````````````````````````````` example
+*[foo*](/uri)
+.
+
    *foo*
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar](baz*)
+.
+
    foo *bar
    
+````````````````````````````````
+
+
+Note that brackets that *aren't* part of links do not take
+precedence:
+
+```````````````````````````````` example
+*foo [bar* baz]
+.
+
    foo [bar baz]
    
+````````````````````````````````
+
+
+These cases illustrate the precedence of HTML tags, code spans,
+and autolinks over link grouping:
+
+```````````````````````````````` example
+[foo 
+.
+
    [foo 
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo`](/uri)`
+.
+
    [foo](/uri)
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo
+.
+
    [foohttps://example.com/?search=](uri)
    
+````````````````````````````````
+
+
+There are three kinds of [reference link](@)s:
+[full](#full-reference-link), [collapsed](#collapsed-reference-link),
+and [shortcut](#shortcut-reference-link).
+
+A [full reference link](@)
+consists of a [link text] immediately followed by a [link label]
+that [matches] a [link reference definition] elsewhere in the document.
+
+A [link label](@)  begins with a left bracket (`[`) and ends
+with the first right bracket (`]`) that is not backslash-escaped.
+Between these brackets there must be at least one character that is not a space,
+tab, or line ending.
+Unescaped square bracket characters are not allowed inside the
+opening and closing square brackets of [link labels].  A link
+label can have at most 999 characters inside the square
+brackets.
+
+One label [matches](@)
+another just in case their normalized forms are equal.  To normalize a
+label, strip off the opening and closing brackets,
+perform the *Unicode case fold*, strip leading and trailing
+spaces, tabs, and line endings, and collapse consecutive internal
+spaces, tabs, and line endings to a single space.  If there are multiple
+matching reference link definitions, the one that comes first in the
+document is used.  (It is desirable in such cases to emit a warning.)
+
+The link's URI and title are provided by the matching [link
+reference definition].
+
+Here is a simple example:
+
+```````````````````````````````` example
+[foo][bar]
+
+[bar]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+The rules for the [link text] are the same as with
+[inline links].  Thus:
+
+The link text may contain balanced brackets, but not unbalanced ones,
+unless they are escaped:
+
+```````````````````````````````` example
+[link [foo [bar]]][ref]
+
+[ref]: /uri
+.
+
    link [foo [bar]]
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[link \[bar][ref]
+
+[ref]: /uri
+.
+
    link [bar
    
+````````````````````````````````
+
+
+The link text may contain inline content:
+
+```````````````````````````````` example
+[link *foo **bar** `#`*][ref]
+
+[ref]: /uri
+.
+
    link foo bar #
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[![moon](moon.jpg)][ref]
+
+[ref]: /uri
+.
+
    moon
    
+````````````````````````````````
+
+
+However, links cannot contain other links, at any level of nesting.
+
+```````````````````````````````` example
+[foo [bar](/uri)][ref]
+
+[ref]: /uri
+.
+
    [foo bar]ref
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar [baz][ref]*][ref]
+
+[ref]: /uri
+.
+
    [foo bar baz]ref
    
+````````````````````````````````
+
+
+(In the examples above, we have two [shortcut reference links]
+instead of one [full reference link].)
+
+The following cases illustrate the precedence of link text grouping over
+emphasis grouping:
+
+```````````````````````````````` example
+*[foo*][ref]
+
+[ref]: /uri
+.
+
    *foo*
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar][ref]*
+
+[ref]: /uri
+.
+
    foo *bar*
    
+````````````````````````````````
+
+
+These cases illustrate the precedence of HTML tags, code spans,
+and autolinks over link grouping:
+
+```````````````````````````````` example
+[foo 
+
+[ref]: /uri
+.
+
    [foo 
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo`][ref]`
+
+[ref]: /uri
+.
+
    [foo][ref]
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo
+
+[ref]: /uri
+.
+
    [foohttps://example.com/?search=][ref]
    
+````````````````````````````````
+
+
+Matching is case-insensitive:
+
+```````````````````````````````` example
+[foo][BaR]
+
+[bar]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+Unicode case fold is used:
+
+```````````````````````````````` example
+[ẞ]
+
+[SS]: /url
+.
+
    ẞ
    
+````````````````````````````````
+
+
+Consecutive internal spaces, tabs, and line endings are treated as one space for
+purposes of determining matching:
+
+```````````````````````````````` example
+[Foo
+  bar]: /url
+
+[Baz][Foo bar]
+.
+
    Baz
    
+````````````````````````````````
+
+
+No spaces, tabs, or line endings are allowed between the [link text] and the
+[link label]:
+
+```````````````````````````````` example
+[foo] [bar]
+
+[bar]: /url "title"
+.
+
    [foo] bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo]
+[bar]
+
+[bar]: /url "title"
+.
+
    [foo]
+bar
    
+````````````````````````````````
+
+
+This is a departure from John Gruber's original Markdown syntax
+description, which explicitly allows whitespace between the link
+text and the link label.  It brings reference links in line with
+[inline links], which (according to both original Markdown and
+this spec) cannot have whitespace after the link text.  More
+importantly, it prevents inadvertent capture of consecutive
+[shortcut reference links]. If whitespace is allowed between the
+link text and the link label, then in the following we will have
+a single reference link, not two shortcut reference links, as
+intended:
+
+``` markdown
+[foo]
+[bar]
+
+[foo]: /url1
+[bar]: /url2
+```
+
+(Note that [shortcut reference links] were introduced by Gruber
+himself in a beta version of `Markdown.pl`, but never included
+in the official syntax description.  Without shortcut reference
+links, it is harmless to allow space between the link text and
+link label; but once shortcut references are introduced, it is
+too dangerous to allow this, as it frequently leads to
+unintended results.)
+
+When there are multiple matching [link reference definitions],
+the first is used:
+
+```````````````````````````````` example
+[foo]: /url1
+
+[foo]: /url2
+
+[bar][foo]
+.
+
    bar
    
+````````````````````````````````
+
+
+Note that matching is performed on normalized strings, not parsed
+inline content.  So the following does not match, even though the
+labels define equivalent inline content:
+
+```````````````````````````````` example
+[bar][foo\!]
+
+[foo!]: /url
+.
+
    [bar][foo!]
    
+````````````````````````````````
+
+
+[Link labels] cannot contain brackets, unless they are
+backslash-escaped:
+
+```````````````````````````````` example
+[foo][ref[]
+
+[ref[]: /uri
+.
+
    [foo][ref[]
    
+
    [ref[]: /uri
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo][ref[bar]]
+
+[ref[bar]]: /uri
+.
+
    [foo][ref[bar]]
    
+
    [ref[bar]]: /uri
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[[[foo]]]
+
+[[[foo]]]: /url
+.
+
    [[[foo]]]
    
+
    [[[foo]]]: /url
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo][ref\[]
+
+[ref\[]: /uri
+.
+
    foo
    
+````````````````````````````````
+
+
+Note that in this example `]` is not backslash-escaped:
+
+```````````````````````````````` example
+[bar\\]: /uri
+
+[bar\\]
+.
+
    bar\
    
+````````````````````````````````
+
+
+A [link label] must contain at least one character that is not a space, tab, or
+line ending:
+
+```````````````````````````````` example
+[]
+
+[]: /uri
+.
+
    []
    
+
    []: /uri
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[
+ ]
+
+[
+ ]: /uri
+.
+
    [
+]
    
+
    [
+]: /uri
    
+````````````````````````````````
+
+
+A [collapsed reference link](@)
+consists of a [link label] that [matches] a
+[link reference definition] elsewhere in the
+document, followed by the string `[]`.
+The contents of the link label are parsed as inlines,
+which are used as the link's text.  The link's URI and title are
+provided by the matching reference link definition.  Thus,
+`[foo][]` is equivalent to `[foo][foo]`.
+
+```````````````````````````````` example
+[foo][]
+
+[foo]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[*foo* bar][]
+
+[*foo* bar]: /url "title"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+[Foo][]
+
+[foo]: /url "title"
+.
+
    Foo
    
+````````````````````````````````
+
+
+
+As with full reference links, spaces, tabs, or line endings are not
+allowed between the two sets of brackets:
+
+```````````````````````````````` example
+[foo] 
+[]
+
+[foo]: /url "title"
+.
+
    foo
+[]
    
+````````````````````````````````
+
+
+A [shortcut reference link](@)
+consists of a [link label] that [matches] a
+[link reference definition] elsewhere in the
+document and is not followed by `[]` or a link label.
+The contents of the link label are parsed as inlines,
+which are used as the link's text.  The link's URI and title
+are provided by the matching link reference definition.
+Thus, `[foo]` is equivalent to `[foo][]`.
+
+```````````````````````````````` example
+[foo]
+
+[foo]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[*foo* bar]
+
+[*foo* bar]: /url "title"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[[*foo* bar]]
+
+[*foo* bar]: /url "title"
+.
+
    [foo bar]
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+[[bar [foo]
+
+[foo]: /url
+.
+
    [[bar foo
    
+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+[Foo]
+
+[foo]: /url "title"
+.
+
    Foo
    
+````````````````````````````````
+
+
+A space after the link text should be preserved:
+
+```````````````````````````````` example
+[foo] bar
+
+[foo]: /url
+.
+
    foo bar
    
+````````````````````````````````
+
+
+If you just want bracketed text, you can backslash-escape the
+opening bracket to avoid links:
+
+```````````````````````````````` example
+\[foo]
+
+[foo]: /url "title"
+.
+
    [foo]
    
+````````````````````````````````
+
+
+Note that this is a link, because a link label ends with the first
+following closing bracket:
+
+```````````````````````````````` example
+[foo*]: /url
+
+*[foo*]
+.
+
    *foo*
    
+````````````````````````````````
+
+
+Full and collapsed references take precedence over shortcut
+references:
+
+```````````````````````````````` example
+[foo][bar]
+
+[foo]: /url1
+[bar]: /url2
+.
+
    foo
    
+````````````````````````````````
+
+```````````````````````````````` example
+[foo][]
+
+[foo]: /url1
+.
+
    foo
    
+````````````````````````````````
+
+Inline links also take precedence:
+
+```````````````````````````````` example
+[foo]()
+
+[foo]: /url1
+.
+
    foo
    
+````````````````````````````````
+
+```````````````````````````````` example
+[foo](not a link)
+
+[foo]: /url1
+.
+
    foo(not a link)
    
+````````````````````````````````
+
+In the following case `[bar][baz]` is parsed as a reference,
+`[foo]` as normal text:
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url
+.
+
    [foo]bar
    
+````````````````````````````````
+
+
+Here, though, `[foo][bar]` is parsed as a reference, since
+`[bar]` is defined:
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url1
+[bar]: /url2
+.
+
    foobaz
    
+````````````````````````````````
+
+
+Here `[foo]` is not parsed as a shortcut reference, because it
+is followed by a link label (even though `[bar]` is not defined):
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url1
+[foo]: /url2
+.
+
    [foo]bar
    
+````````````````````````````````
+
+
+
+## Images
+
+Syntax for images is like the syntax for links, with one
+difference. Instead of [link text], we have an
+[image description](@).  The rules for this are the
+same as for [link text], except that (a) an
+image description starts with `![` rather than `[`, and
+(b) an image description may contain links.
+An image description has inline elements
+as its contents.  When an image is rendered to HTML,
+this is standardly used as the image's `alt` attribute.
+
+```````````````````````````````` example
+![foo](/url "title")
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo *bar*]
+
+[foo *bar*]: train.jpg "train & tracks"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo ![bar](/url)](/url2)
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo [bar](/url)](/url2)
+.
+
    foo bar
    
+````````````````````````````````
+
+
+Though this spec is concerned with parsing, not rendering, it is
+recommended that in rendering to HTML, only the plain string content
+of the [image description] be used.  Note that in
+the above example, the alt attribute's value is `foo bar`, not `foo
+[bar](/url)` or `foo bar`.  Only the plain string
+content is rendered, without formatting.
+
+```````````````````````````````` example
+![foo *bar*][]
+
+[foo *bar*]: train.jpg "train & tracks"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo *bar*][foobar]
+
+[FOOBAR]: train.jpg "train & tracks"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo](train.jpg)
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+My ![foo bar](/path/to/train.jpg  "title"   )
+.
+
    My foo bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo]()
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![](/url)
+.
+
    
+````````````````````````````````
+
+
+Reference-style:
+
+```````````````````````````````` example
+![foo][bar]
+
+[bar]: /url
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo][bar]
+
+[BAR]: /url
+.
+
    foo
    
+````````````````````````````````
+
+
+Collapsed:
+
+```````````````````````````````` example
+![foo][]
+
+[foo]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![*foo* bar][]
+
+[*foo* bar]: /url "title"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+The labels are case-insensitive:
+
+```````````````````````````````` example
+![Foo][]
+
+[foo]: /url "title"
+.
+
    Foo
    
+````````````````````````````````
+
+
+As with reference links, spaces, tabs, and line endings, are not allowed
+between the two sets of brackets:
+
+```````````````````````````````` example
+![foo] 
+[]
+
+[foo]: /url "title"
+.
+
    foo
+[]
    
+````````````````````````````````
+
+
+Shortcut:
+
+```````````````````````````````` example
+![foo]
+
+[foo]: /url "title"
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+![*foo* bar]
+
+[*foo* bar]: /url "title"
+.
+
    foo bar
    
+````````````````````````````````
+
+
+Note that link labels cannot contain unescaped brackets:
+
+```````````````````````````````` example
+![[foo]]
+
+[[foo]]: /url "title"
+.
+
    ![[foo]]
    
+
    [[foo]]: /url "title"
    
+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+![Foo]
+
+[foo]: /url "title"
+.
+
    Foo
    
+````````````````````````````````
+
+
+If you just want a literal `!` followed by bracketed text, you can
+backslash-escape the opening `[`:
+
+```````````````````````````````` example
+!\[foo]
+
+[foo]: /url "title"
+.
+
    ![foo]
    
+````````````````````````````````
+
+
+If you want a link after a literal `!`, backslash-escape the
+`!`:
+
+```````````````````````````````` example
+\![foo]
+
+[foo]: /url "title"
+.
+
    !foo
    
+````````````````````````````````
+
+
+## Autolinks
+
+[Autolink](@)s are absolute URIs and email addresses inside
+`<` and `>`. They are parsed as links, with the URL or email address
+as the link label.
+
+A [URI autolink](@) consists of `<`, followed by an
+[absolute URI] followed by `>`.  It is parsed as
+a link to the URI, with the URI as the link's label.
+
+An [absolute URI](@),
+for these purposes, consists of a [scheme] followed by a colon (`:`)
+followed by zero or more characters other than [ASCII control
+characters][ASCII control character], [space], `<`, and `>`.
+If the URI includes these characters, they must be percent-encoded
+(e.g. `%20` for a space).
+
+For purposes of this spec, a [scheme](@) is any sequence
+of 2--32 characters beginning with an ASCII letter and followed
+by any combination of ASCII letters, digits, or the symbols plus
+("+"), period ("."), or hyphen ("-").
+
+Here are some valid autolinks:
+
+```````````````````````````````` example
+
+.
+
    http://foo.bar.baz
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    https://foo.bar.baz/test?q=hello&id=22&boolean
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    irc://foo.bar:2233/baz
    
+````````````````````````````````
+
+
+Uppercase is also fine:
+
+```````````````````````````````` example
+
+.
+
    MAILTO:FOO@BAR.BAZ
    
+````````````````````````````````
+
+
+Note that many strings that count as [absolute URIs] for
+purposes of this spec are not valid URIs, because their
+schemes are not registered or because of other problems
+with their syntax:
+
+```````````````````````````````` example
+
+.
+
    a+b+c:d
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    made-up-scheme://foo,bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    https://../
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    localhost:5001/foo
    
+````````````````````````````````
+
+
+Spaces are not allowed in autolinks:
+
+```````````````````````````````` example
+
+.
+
    <https://foo.bar/baz bim>
    
+````````````````````````````````
+
+
+Backslash-escapes do not work inside autolinks:
+
+```````````````````````````````` example
+
+.
+
    https://example.com/\[\
    
+````````````````````````````````
+
+
+An [email autolink](@)
+consists of `<`, followed by an [email address],
+followed by `>`.  The link's label is the email address,
+and the URL is `mailto:` followed by the email address.
+
+An [email address](@),
+for these purposes, is anything that matches
+the [non-normative regex from the HTML5
+spec](https://html.spec.whatwg.org/multipage/forms.html#e-mail-state-(type=email)):
+
+    /^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?
+    (?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
+
+Examples of email autolinks:
+
+```````````````````````````````` example
+
+.
+
    foo@bar.example.com
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    foo+special@Bar.baz-bar0.com
    
+````````````````````````````````
+
+
+Backslash-escapes do not work inside email autolinks:
+
+```````````````````````````````` example
+
+.
+
    <foo+@bar.example.com>
    
+````````````````````````````````
+
+
+These are not autolinks:
+
+```````````````````````````````` example
+<>
+.
+
    <>
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+< https://foo.bar >
+.
+
    < https://foo.bar >
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    <m:abc>
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    <foo.bar.baz>
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+https://example.com
+.
+
    https://example.com
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo@bar.example.com
+.
+
    foo@bar.example.com
    
+````````````````````````````````
+
+
+## Raw HTML
+
+Text between `<` and `>` that looks like an HTML tag is parsed as a
+raw HTML tag and will be rendered in HTML without escaping.
+Tag and attribute names are not limited to current HTML tags,
+so custom tags (and even, say, DocBook tags) may be used.
+
+Here is the grammar for tags:
+
+A [tag name](@) consists of an ASCII letter
+followed by zero or more ASCII letters, digits, or
+hyphens (`-`).
+
+An [attribute](@) consists of spaces, tabs, and up to one line ending,
+an [attribute name], and an optional
+[attribute value specification].
+
+An [attribute name](@)
+consists of an ASCII letter, `_`, or `:`, followed by zero or more ASCII
+letters, digits, `_`, `.`, `:`, or `-`.  (Note:  This is the XML
+specification restricted to ASCII.  HTML5 is laxer.)
+
+An [attribute value specification](@)
+consists of optional spaces, tabs, and up to one line ending,
+a `=` character, optional spaces, tabs, and up to one line ending,
+and an [attribute value].
+
+An [attribute value](@)
+consists of an [unquoted attribute value],
+a [single-quoted attribute value], or a [double-quoted attribute value].
+
+An [unquoted attribute value](@)
+is a nonempty string of characters not
+including spaces, tabs, line endings, `"`, `'`, `=`, `<`, `>`, or `` ` ``.
+
+A [single-quoted attribute value](@)
+consists of `'`, zero or more
+characters not including `'`, and a final `'`.
+
+A [double-quoted attribute value](@)
+consists of `"`, zero or more
+characters not including `"`, and a final `"`.
+
+An [open tag](@) consists of a `<` character, a [tag name],
+zero or more [attributes], optional spaces, tabs, and up to one line ending,
+an optional `/` character, and a `>` character.
+
+A [closing tag](@) consists of the string ``.
+
+An [HTML comment](@) consists of ``, ``, or  ``, and `-->` (see the
+[HTML spec](https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state)).
+
+A [processing instruction](@)
+consists of the string ``, and the string
+`?>`.
+
+A [declaration](@) consists of the string ``, and the character `>`.
+
+A [CDATA section](@) consists of
+the string ``, and the string `]]>`.
+
+An [HTML tag](@) is an [open tag], a [closing tag],
+an [HTML comment], a [processing instruction], a [declaration],
+or a [CDATA section].
+
+Here are some simple open tags:
+
+```````````````````````````````` example
+
+.
+
    
+````````````````````````````````
+
+
+Empty elements:
+
+```````````````````````````````` example
+
+.
+
    
+````````````````````````````````
+
+
+Whitespace is allowed:
+
+```````````````````````````````` example
+
+.
+
    
+````````````````````````````````
+
+
+With attributes:
+
+```````````````````````````````` example
+
+.
+
    
+````````````````````````````````
+
+
+Custom tag names can be used:
+
+```````````````````````````````` example
+Foo 
+.
+
    Foo 
    
+````````````````````````````````
+
+
+Illegal tag names, not parsed as HTML:
+
+```````````````````````````````` example
+<33> <__>
+.
+
    <33> <__>
    
+````````````````````````````````
+
+
+Illegal attribute names:
+
+```````````````````````````````` example
+    
+.
+
    <a h*#ref="hi">
    
+````````````````````````````````
+
+
+Illegal attribute values:
+
+```````````````````````````````` example
+    
+.
+
    </a href="foo">
    
+````````````````````````````````
+
+
+Comments:
+
+```````````````````````````````` example
+foo 
+.
+
    foo 
    
+````````````````````````````````
+
+```````````````````````````````` example
+foo  foo -->
+
+foo  foo -->
+.
+
    foo  foo -->
    
+
    foo  foo -->
    
+````````````````````````````````
+
+
+Processing instructions:
+
+```````````````````````````````` example
+foo 
+.
+
    foo 
    
+````````````````````````````````
+
+
+Declarations:
+
+```````````````````````````````` example
+foo 
+.
+
    foo 
    
+````````````````````````````````
+
+
+CDATA sections:
+
+```````````````````````````````` example
+foo &<]]>
+.
+
    foo &<]]>
    
+````````````````````````````````
+
+
+Entity and numeric character references are preserved in HTML
+attributes:
+
+```````````````````````````````` example
+foo     
+.
+
    foo 
    
+````````````````````````````````
+
+
+Backslash escapes do not work in HTML attributes:
+
+```````````````````````````````` example
+foo 
+.
+
    foo 
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    <a href=""">
    
+````````````````````````````````
+
+
+A block quote can prevent a line from being parsed as inline HTML,
+even though line breaks are allowed in tags:
+
+```````````````````````````````` example
+     quoted text
+.
+
    <a
    
+
    
+
    quoted text
    
+
    
+````````````````````````````````
+
+
+## Hard line breaks
+
+A line ending (not in a code span or HTML tag) that is preceded
+by two or more spaces and does not occur at the end of a block
+is parsed as a [hard line break](@) (rendered
+in HTML as a `
    ` tag):
+
+```````````````````````````````` example
+foo  
+baz
+.
+
    foo
    
+baz
    
+````````````````````````````````
+
+
+For a more visible alternative, a backslash before the
+[line ending] may be used instead of two or more spaces:
+
+```````````````````````````````` example
+foo\
+baz
+.
+
    foo
    
+baz
    
+````````````````````````````````
+
+
+More than two spaces can be used:
+
+```````````````````````````````` example
+foo       
+baz
+.
+
    foo
    
+baz
    
+````````````````````````````````
+
+
+Leading spaces at the beginning of the next line are ignored:
+
+```````````````````````````````` example
+foo  
+     bar
+.
+
    foo
    
+bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo\
+     bar
+.
+
    foo
    
+bar
    
+````````````````````````````````
+
+
+Hard line breaks can occur inside emphasis, links, and other constructs
+that allow inline content:
+
+```````````````````````````````` example
+*foo  
+bar*
+.
+
    foo
    
+bar
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo\
+bar*
+.
+
    foo
    
+bar
    
+````````````````````````````````
+
+
+Hard line breaks do not occur inside code spans
+
+```````````````````````````````` example
+`code  
+span`
+.
+
    code   span
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+`code\
+span`
+.
+
    code\ span
    
+````````````````````````````````
+
+
+or HTML tags:
+
+```````````````````````````````` example
+    
+.
+
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
    
+````````````````````````````````
+
+
+Hard line breaks are for separating inline content within a block.
+Neither syntax for hard line breaks works at the end of a paragraph or
+other block element:
+
+```````````````````````````````` example
+foo\
+.
+
    foo\
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+foo  
+.
+
    foo
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+### foo\
+.
+
    foo\
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+### foo  
+.
+
    foo
    
+````````````````````````````````
+
+
+## Soft line breaks
+
+A regular line ending (not in a code span or HTML tag) that is not
+preceded by two or more spaces or a backslash is parsed as a
+[softbreak](@).  (A soft line break may be rendered in HTML either as a
+[line ending] or as a space. The result will be the same in
+browsers. In the examples here, a [line ending] will be used.)
+
+```````````````````````````````` example
+foo
+baz
+.
+
    foo
+baz
    
+````````````````````````````````
+
+
+Spaces at the end of the line and beginning of the next line are
+removed:
+
+```````````````````````````````` example
+foo 
+ baz
+.
+
    foo
+baz
    
+````````````````````````````````
+
+
+A conforming parser may render a soft line break in HTML either as a
+line ending or as a space.
+
+A renderer may also provide an option to render soft line breaks
+as hard line breaks.
+
+## Textual content
+
+Any characters not given an interpretation by the above rules will
+be parsed as plain textual content.
+
+```````````````````````````````` example
+hello $.;'there
+.
+
    hello $.;'there
    
+````````````````````````````````
+
+
+```````````````````````````````` example
+Foo χρῆν
+.
+
    Foo χρῆν
    
+````````````````````````````````
+
+
+Internal spaces are preserved verbatim:
+
+```````````````````````````````` example
+Multiple     spaces
+.
+
    Multiple     spaces
    
+````````````````````````````````
+
+
+
+
+# Appendix: A parsing strategy
+
+In this appendix we describe some features of the parsing strategy
+used in the CommonMark reference implementations.
+
+## Overview
+
+Parsing has two phases:
+
+1. In the first phase, lines of input are consumed and the block
+structure of the document---its division into paragraphs, block quotes,
+list items, and so on---is constructed.  Text is assigned to these
+blocks but not parsed. Link reference definitions are parsed and a
+map of links is constructed.
+
+2. In the second phase, the raw text contents of paragraphs and headings
+are parsed into sequences of Markdown inline elements (strings,
+code spans, links, emphasis, and so on), using the map of link
+references constructed in phase 1.
+
+At each point in processing, the document is represented as a tree of
+**blocks**.  The root of the tree is a `document` block.  The `document`
+may have any number of other blocks as **children**.  These children
+may, in turn, have other blocks as children.  The last child of a block
+is normally considered **open**, meaning that subsequent lines of input
+can alter its contents.  (Blocks that are not open are **closed**.)
+Here, for example, is a possible document tree, with the open blocks
+marked by arrows:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+         list_item
+           paragraph
+             "Qui *quodsi iracundia*"
+      -> list_item
+        -> paragraph
+             "aliquando id"
+```
+
+## Phase 1: block structure
+
+Each line that is processed has an effect on this tree.  The line is
+analyzed and, depending on its contents, the document may be altered
+in one or more of the following ways:
+
+1. One or more open blocks may be closed.
+2. One or more new blocks may be created as children of the
+   last open block.
+3. Text may be added to the last (deepest) open block remaining
+   on the tree.
+
+Once a line has been incorporated into the tree in this way,
+it can be discarded, so input can be read in a stream.
+
+For each line, we follow this procedure:
+
+1. First we iterate through the open blocks, starting with the
+root document, and descending through last children down to the last
+open block.  Each block imposes a condition that the line must satisfy
+if the block is to remain open.  For example, a block quote requires a
+`>` character.  A paragraph requires a non-blank line.
+In this phase we may match all or just some of the open
+blocks.  But we cannot close unmatched blocks yet, because we may have a
+[lazy continuation line].
+
+2.  Next, after consuming the continuation markers for existing
+blocks, we look for new block starts (e.g. `>` for a block quote).
+If we encounter a new block start, we close any blocks unmatched
+in step 1 before creating the new block as a child of the last
+matched container block.
+
+3.  Finally, we look at the remainder of the line (after block
+markers like `>`, list markers, and indentation have been consumed).
+This is text that can be incorporated into the last open
+block (a paragraph, code block, heading, or raw HTML).
+
+Setext headings are formed when we see a line of a paragraph
+that is a [setext heading underline].
+
+Reference link definitions are detected when a paragraph is closed;
+the accumulated text lines are parsed to see if they begin with
+one or more reference link definitions.  Any remainder becomes a
+normal paragraph.
+
+We can see how this works by considering how the tree above is
+generated by four lines of Markdown:
+
+``` markdown
+> Lorem ipsum dolor
+sit amet.
+> - Qui *quodsi iracundia*
+> - aliquando id
+```
+
+At the outset, our document model is just
+
+``` tree
+-> document
+```
+
+The first line of our text,
+
+``` markdown
+> Lorem ipsum dolor
+```
+
+causes a `block_quote` block to be created as a child of our
+open `document` block, and a `paragraph` block as a child of
+the `block_quote`.  Then the text is added to the last open
+block, the `paragraph`:
+
+``` tree
+-> document
+  -> block_quote
+    -> paragraph
+         "Lorem ipsum dolor"
+```
+
+The next line,
+
+``` markdown
+sit amet.
+```
+
+is a "lazy continuation" of the open `paragraph`, so it gets added
+to the paragraph's text:
+
+``` tree
+-> document
+  -> block_quote
+    -> paragraph
+         "Lorem ipsum dolor\nsit amet."
+```
+
+The third line,
+
+``` markdown
+> - Qui *quodsi iracundia*
+```
+
+causes the `paragraph` block to be closed, and a new `list` block
+opened as a child of the `block_quote`.  A `list_item` is also
+added as a child of the `list`, and a `paragraph` as a child of
+the `list_item`.  The text is then added to the new `paragraph`:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+      -> list_item
+        -> paragraph
+             "Qui *quodsi iracundia*"
+```
+
+The fourth line,
+
+``` markdown
+> - aliquando id
+```
+
+causes the `list_item` (and its child the `paragraph`) to be closed,
+and a new `list_item` opened up as child of the `list`.  A `paragraph`
+is added as a child of the new `list_item`, to contain the text.
+We thus obtain the final tree:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+         list_item
+           paragraph
+             "Qui *quodsi iracundia*"
+      -> list_item
+        -> paragraph
+             "aliquando id"
+```
+
+## Phase 2: inline structure
+
+Once all of the input has been parsed, all open blocks are closed.
+
+We then "walk the tree," visiting every node, and parse raw
+string contents of paragraphs and headings as inlines.  At this
+point we have seen all the link reference definitions, so we can
+resolve reference links as we go.
+
+``` tree
+document
+  block_quote
+    paragraph
+      str "Lorem ipsum dolor"
+      softbreak
+      str "sit amet."
+    list (type=bullet tight=true bullet_char=-)
+      list_item
+        paragraph
+          str "Qui "
+          emph
+            str "quodsi iracundia"
+      list_item
+        paragraph
+          str "aliquando id"
+```
+
+Notice how the [line ending] in the first paragraph has
+been parsed as a `softbreak`, and the asterisks in the first list item
+have become an `emph`.
+
+### An algorithm for parsing nested emphasis and links
+
+By far the trickiest part of inline parsing is handling emphasis,
+strong emphasis, links, and images.  This is done using the following
+algorithm.
+
+When we're parsing inlines and we hit either
+
+- a run of `*` or `_` characters, or
+- a `[` or `![`
+
+we insert a text node with these symbols as its literal content, and we
+add a pointer to this text node to the [delimiter stack](@).
+
+The [delimiter stack] is a doubly linked list.  Each
+element contains a pointer to a text node, plus information about
+
+- the type of delimiter (`[`, `![`, `*`, `_`)
+- the number of delimiters,
+- whether the delimiter is "active" (all are active to start), and
+- whether the delimiter is a potential opener, a potential closer,
+  or both (which depends on what sort of characters precede
+  and follow the delimiters).
+
+When we hit a `]` character, we call the *look for link or image*
+procedure (see below).
+
+When we hit the end of the input, we call the *process emphasis*
+procedure (see below), with `stack_bottom` = NULL.
+
+#### *look for link or image*
+
+Starting at the top of the delimiter stack, we look backwards
+through the stack for an opening `[` or `![` delimiter.
+
+- If we don't find one, we return a literal text node `]`.
+
+- If we do find one, but it's not *active*, we remove the inactive
+  delimiter from the stack, and return a literal text node `]`.
+
+- If we find one and it's active, then we parse ahead to see if
+  we have an inline link/image, reference link/image, collapsed reference
+  link/image, or shortcut reference link/image.
+
+  + If we don't, then we remove the opening delimiter from the
+    delimiter stack and return a literal text node `]`.
+
+  + If we do, then
+
+    * We return a link or image node whose children are the inlines
+      after the text node pointed to by the opening delimiter.
+
+    * We run *process emphasis* on these inlines, with the `[` opener
+      as `stack_bottom`.
+
+    * We remove the opening delimiter.
+
+    * If we have a link (and not an image), we also set all
+      `[` delimiters before the opening delimiter to *inactive*.  (This
+      will prevent us from getting links within links.)
+
+#### *process emphasis*
+
+Parameter `stack_bottom` sets a lower bound to how far we
+descend in the [delimiter stack].  If it is NULL, we can
+go all the way to the bottom.  Otherwise, we stop before
+visiting `stack_bottom`.
+
+Let `current_position` point to the element on the [delimiter stack]
+just above `stack_bottom` (or the first element if `stack_bottom`
+is NULL).
+
+We keep track of the `openers_bottom` for each delimiter
+type (`*`, `_`), indexed to the length of the closing delimiter run
+(modulo 3) and to whether the closing delimiter can also be an
+opener.  Initialize this to `stack_bottom`.
+
+Then we repeat the following until we run out of potential
+closers:
+
+- Move `current_position` forward in the delimiter stack (if needed)
+  until we find the first potential closer with delimiter `*` or `_`.
+  (This will be the potential closer closest
+  to the beginning of the input -- the first one in parse order.)
+
+- Now, look back in the stack (staying above `stack_bottom` and
+  the `openers_bottom` for this delimiter type) for the
+  first valid potential opener, where being "valid" requires that:
+
+  + the token is a potential opener; and
+
+  + the token has the same delimiter type as the current potential
+    closer; and
+
+  + any of the following are true:
+
+    * the closer is not a potential opener and the opener is not a
+      potential closer; or
+
+    * the original length of the closing delimiter run is a multiple
+      of 3; or
+
+    * the original length of the opening delimiter run + the original
+      length of the closing delimiter run is not a multiple of 3
+
+- If one is found:
+
+  + Figure out whether we have emphasis or strong emphasis:
+    if both closer and opener spans have length >= 2, we have
+    strong, otherwise regular.
+
+  + Insert an emph or strong emph node accordingly, after
+    the text node corresponding to the opener.
+
+  + Remove any delimiters between the opener and closer from
+    the delimiter stack.
+
+  + Remove 1 (for regular emph) or 2 (for strong emph) delimiters
+    from the opening and closing text nodes.  If they become empty
+    as a result, remove them and remove the corresponding element
+    of the delimiter stack.  If the closing node is removed, reset
+    `current_position` to the next element in the stack.
+
+- If none is found:
+
+  + Set `openers_bottom` to the element before `current_position`.
+    (We know that there are no openers for this kind of closer up to and
+    including this point, so this puts a lower bound on future searches.)
+
+  + If the closer at `current_position` is not a potential opener,
+    remove it from the delimiter stack (since we know it can't
+    be a closer either).
+
+  + Advance `current_position` to the next element in the stack.
+
+After we're done, we remove all delimiters above `stack_bottom` from the
+delimiter stack.
diff --git a/tests/server_interactive/semanticTokensMarkdownDocs.lean b/tests/server_interactive/semanticTokensMarkdownDocs.lean
new file mode 100644
index 000000000000..3f20eb0acf68
--- /dev/null
+++ b/tests/server_interactive/semanticTokensMarkdownDocs.lean
@@ -0,0 +1,103 @@
+/-!
+This test exercises plain (non-Verso) Markdown docstring highlighting. It
+includes at least one instance of every CommonMark construct the Markdown
+tokenizer recognizes and at least one instance of every `markup*` token
+type the server emits — including emphasis composed with each block
+context (heading, blockquote, list).
+-/
+
+/--
+# Heading 1
+
+## Heading 2 with **bold** and *italic*
+
+### Heading 3 with ***both***
+
+Setext heading
+==============
+
+Paragraph with **bold**, *italic*, ***both***, `inline code`, a [link](https://example.com), an
+![image](https://example.com/x.png "title"), an autolink , and a hard break on
+the next line.
+
+---
+
+> Blockquote with **bold**, *italic*, and ***both***.
+>
+> > Nested blockquote.
+>
+> * List inside blockquote with *italic*
+
+* Bullet list with **bold**
+* Another item with *italic* and ***both***
+
+1. Ordered with `code`
+2. Another
+
+```lean
+example : True := trivial
+```
+
+    indented
+    code block
+
+[ref]: https://example.com "title"
+
+A reference link: [text][ref] and a shortcut: [ref]. And a non-reference: [text][bogus]. And a
+forward ref: [text][ref2]
+
+[ref2]: https://example.com
+
+A paragraph with *cross-line italic* and **cross-line bold** to verify multi-line emphasis is
+recognised.
+-/
+def x := ()
+
+/-!
+# Heading 1
+
+## Heading 2 with **bold** and *italic*
+
+### Heading 3 with ***both***
+
+Setext heading
+==============
+
+Paragraph with **bold**, *italic*, ***both***, `inline code`, a [link](https://example.com), an
+![image **with embedded content** [link[nonlink][ref]][ref]](https://example.com/x.png "title"), an
+autolink , and a hard break on the next line.
+
+---
+
+> Blockquote with **bold**, *italic*, and ***both***.
+>
+> > Nested blockquote.
+>
+> * List inside blockquote with *italic*
+
+* Bullet list with **bold**
+* Another item with *italic* and ***both***
+
+1. Ordered with `code`
+2. Another
+
+```lean
+example : True := trivial
+```
+
+    indented
+    code block
+
+[ref]: https://example.com "title"
+
+A reference link: [text][ref] and a shortcut: [ref]. And a non-reference: [text][bogus]. And a
+forward ref: [text][ref2]
+
+[ref2]: https://example.com
+
+A paragraph with *cross-line italic* and **cross-line bold** to verify multi-line emphasis is
+recognised.
+-/
+
+--^ collectDiagnostics
+--^ textDocument/semanticTokens/full
diff --git a/tests/server_interactive/semanticTokensMarkdownDocs.lean.out.expected b/tests/server_interactive/semanticTokensMarkdownDocs.lean.out.expected
new file mode 100644
index 000000000000..15e601b71385
--- /dev/null
+++ b/tests/server_interactive/semanticTokensMarkdownDocs.lean.out.expected
@@ -0,0 +1,1712 @@
+{"version": 1,
+ "uri": "file:///semanticTokensMarkdownDocs.lean",
+ "isIncremental": false,
+ "diagnostics": []}
+{"textDocument": {"uri": "file:///semanticTokensMarkdownDocs.lean"},
+ "position": {"line": 100, "character": 2}}
+{"data":
+ [1,
+  0,
+  73,
+  41,
+  0,
+  1,
+  0,
+  73,
+  41,
+  0,
+  1,
+  0,
+  56,
+  41,
+  0,
+  0,
+  56,
+  1,
+  0,
+  0,
+  0,
+  1,
+  7,
+  39,
+  0,
+  0,
+  7,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  41,
+  0,
+  1,
+  0,
+  67,
+  41,
+  0,
+  1,
+  0,
+  36,
+  41,
+  0,
+  1,
+  0,
+  2,
+  41,
+  0,
+  3,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  9,
+  27,
+  0,
+  2,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  15,
+  27,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  28,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  5,
+  27,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  29,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  0,
+  4,
+  15,
+  27,
+  0,
+  0,
+  15,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  30,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  2,
+  0,
+  14,
+  27,
+  0,
+  1,
+  0,
+  14,
+  0,
+  0,
+  2,
+  0,
+  15,
+  41,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  24,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  25,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  26,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  11,
+  39,
+  0,
+  0,
+  11,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  19,
+  43,
+  0,
+  0,
+  19,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  41,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  25,
+  43,
+  0,
+  0,
+  27,
+  5,
+  18,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  14,
+  41,
+  0,
+  0,
+  14,
+  1,
+  0,
+  0,
+  0,
+  1,
+  19,
+  43,
+  0,
+  0,
+  19,
+  1,
+  0,
+  0,
+  0,
+  1,
+  21,
+  41,
+  0,
+  1,
+  0,
+  14,
+  41,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  16,
+  31,
+  0,
+  0,
+  16,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  32,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  2,
+  31,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  33,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  31,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  34,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  31,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  2,
+  18,
+  31,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  2,
+  28,
+  35,
+  0,
+  0,
+  28,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  37,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  17,
+  35,
+  0,
+  0,
+  17,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  36,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  18,
+  35,
+  0,
+  0,
+  18,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  37,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  35,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  38,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  2,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  13,
+  35,
+  0,
+  0,
+  13,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  39,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  1,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  7,
+  35,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  0,
+  3,
+  4,
+  3,
+  0,
+  1,
+  0,
+  25,
+  40,
+  0,
+  1,
+  0,
+  3,
+  0,
+  0,
+  2,
+  0,
+  12,
+  40,
+  0,
+  1,
+  4,
+  10,
+  40,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  2,
+  19,
+  43,
+  0,
+  0,
+  21,
+  5,
+  18,
+  0,
+  2,
+  0,
+  18,
+  41,
+  0,
+  0,
+  18,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  17,
+  41,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  43,
+  41,
+  0,
+  1,
+  0,
+  13,
+  41,
+  0,
+  0,
+  13,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  44,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  44,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  2,
+  19,
+  43,
+  0,
+  2,
+  0,
+  17,
+  41,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  17,
+  25,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  41,
+  0,
+  0,
+  5,
+  2,
+  0,
+  0,
+  0,
+  2,
+  15,
+  24,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  33,
+  41,
+  0,
+  1,
+  0,
+  11,
+  41,
+  0,
+  1,
+  0,
+  2,
+  41,
+  0,
+  1,
+  0,
+  3,
+  0,
+  0,
+  3,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  9,
+  27,
+  0,
+  2,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  15,
+  27,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  28,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  5,
+  27,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  29,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  0,
+  4,
+  15,
+  27,
+  0,
+  0,
+  15,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  30,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  2,
+  0,
+  14,
+  27,
+  0,
+  1,
+  0,
+  14,
+  0,
+  0,
+  2,
+  0,
+  15,
+  41,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  24,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  25,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  26,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  41,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  11,
+  39,
+  0,
+  0,
+  11,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  19,
+  43,
+  0,
+  0,
+  19,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  41,
+  0,
+  0,
+  6,
+  2,
+  0,
+  0,
+  0,
+  2,
+  21,
+  24,
+  0,
+  0,
+  21,
+  2,
+  0,
+  0,
+  0,
+  2,
+  6,
+  41,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  7,
+  41,
+  0,
+  0,
+  7,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  25,
+  43,
+  0,
+  0,
+  27,
+  5,
+  18,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  1,
+  0,
+  9,
+  41,
+  0,
+  0,
+  9,
+  1,
+  0,
+  0,
+  0,
+  1,
+  19,
+  43,
+  0,
+  0,
+  19,
+  1,
+  0,
+  0,
+  0,
+  1,
+  36,
+  41,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  16,
+  31,
+  0,
+  0,
+  16,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  32,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  2,
+  31,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  33,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  31,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  34,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  31,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  2,
+  18,
+  31,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  0,
+  2,
+  28,
+  35,
+  0,
+  0,
+  28,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  37,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  17,
+  35,
+  0,
+  0,
+  17,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  36,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  1,
+  0,
+  1,
+  0,
+  0,
+  0,
+  2,
+  18,
+  35,
+  0,
+  0,
+  18,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  37,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  35,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  2,
+  0,
+  0,
+  0,
+  2,
+  4,
+  38,
+  0,
+  0,
+  4,
+  2,
+  0,
+  0,
+  0,
+  2,
+  1,
+  0,
+  0,
+  2,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  13,
+  35,
+  0,
+  0,
+  13,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  39,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  1,
+  0,
+  2,
+  0,
+  0,
+  0,
+  3,
+  7,
+  35,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  0,
+  3,
+  4,
+  3,
+  0,
+  1,
+  0,
+  25,
+  40,
+  0,
+  1,
+  0,
+  3,
+  0,
+  0,
+  2,
+  0,
+  12,
+  40,
+  0,
+  1,
+  4,
+  10,
+  40,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  2,
+  19,
+  43,
+  0,
+  0,
+  21,
+  5,
+  18,
+  0,
+  2,
+  0,
+  18,
+  41,
+  0,
+  0,
+  18,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  17,
+  41,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  44,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  43,
+  41,
+  0,
+  1,
+  0,
+  13,
+  41,
+  0,
+  0,
+  13,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  41,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  44,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  44,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  2,
+  19,
+  43,
+  0,
+  2,
+  0,
+  17,
+  41,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  17,
+  25,
+  0,
+  0,
+  17,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  41,
+  0,
+  0,
+  5,
+  2,
+  0,
+  0,
+  0,
+  2,
+  15,
+  24,
+  0,
+  0,
+  15,
+  2,
+  0,
+  0,
+  0,
+  2,
+  33,
+  41,
+  0,
+  1,
+  0,
+  11,
+  41,
+  0,
+  1,
+  0,
+  2,
+  41,
+  0]}
diff --git a/tests/server_interactive/semanticTokensVersoDocs.lean b/tests/server_interactive/semanticTokensVersoDocs.lean
index 6be5ff94b9ea..857f1e36971c 100644
--- a/tests/server_interactive/semanticTokensVersoDocs.lean
+++ b/tests/server_interactive/semanticTokensVersoDocs.lean
@@ -1,14 +1,30 @@
 set_option doc.verso true
 /-!
-This test checks that Verso docstring semantic tokens work as expected. In particular, it tests that
-overlapping token handling does what we want, because the unannotated identifiers and the spaces in
-the {lit}`code` elements are assigned the string type, while variables etc are given info-based
-tokens.
+This test checks that Verso docstring semantic tokens work as expected. In particular, it tests
+overlapping token handling, where unannotated identifiers and spaces inside literal-code elements
+receive the string type while variables and similar receive info-based tokens that override string.
+It also checks that plain inline prose receives the markupDocText type, so themes can recolor
+Verso-format docstring body text uniformly as documentation.
 -/
 /-- {name}`foo1` {lean}`foo1 x` {assert}`foo1 4 = 5` -/
 def foo1 (x : Nat) := x.succ
 /-- {name}`foo1` {lean}`foo1 x` {assert}`foo2 = foo1` -/
 def foo2 (x : Nat) := x |>.succ
+
+/--
+Plain prose in a Verso docstring should be tagged as documentation: the leading sentence here, and
+the trailing one, are pure inline text with no role markup, *with one piece of bold* and _one
+piece of italic_, plus *_bold and italic combined_*. The role {lit}`mid` sits between them,
+surrounded by ordinary spaces.
+
+# Heading prose with *bold* and _italic_
+
+> Blockquoted prose with *bold* and _italic_.
+
+* Bullet item with *bold* and _italic_.
+* Another item.
+-/
+def proseExample : Nat := 0
 /--
 *bold* _emph_ *_both_* {lit}`code`
 ```leanTerm
diff --git a/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected b/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
index 236c9a2f483b..c43b3924d404 100644
--- a/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
+++ b/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
@@ -1,9 +1,19 @@
 {"version": 1,
  "uri": "file:///semanticTokensVersoDocs.lean",
  "isIncremental": false,
- "diagnostics": []}
+ "diagnostics":
+ [{"source": "Lean 4",
+   "severity": 1,
+   "range":
+   {"start": {"line": 21, "character": 0},
+    "end": {"line": 21, "character": 45}},
+   "message":
+   "Unsupported syntax: failed to pretty print term (use 'set_option pp.rawOnError true' for raw representation)",
+   "fullRange":
+   {"start": {"line": 21, "character": 0},
+    "end": {"line": 21, "character": 45}}}]}
 {"textDocument": {"uri": "file:///semanticTokensVersoDocs.lean"},
- "position": {"line": 35, "character": 2}}
+ "position": {"line": 51, "character": 2}}
 {"data":
  [0,
   0,
@@ -15,37 +25,32 @@
   4,
   0,
   0,
-  4,
-  4,
-  1,
-  0,
-  0,
-  0,
-  1,
-  3,
-  3,
+  2,
   0,
+  95,
+  41,
   0,
-  3,
   1,
   0,
+  97,
+  41,
   0,
-  0,
-  1,
   1,
   0,
-  0,
+  99,
+  41,
   0,
   1,
-  4,
-  18,
   0,
+  93,
+  41,
   0,
-  4,
   1,
   0,
+  60,
+  41,
   0,
-  3,
+  2,
   4,
   1,
   0,
@@ -76,7 +81,12 @@
   0,
   0,
   0,
-  2,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
   1,
   0,
   0,
@@ -111,7 +121,12 @@
   0,
   0,
   0,
-  2,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
   1,
   0,
   0,
@@ -140,6 +155,11 @@
   1,
   0,
   0,
+  0,
+  1,
+  1,
+  41,
+  0,
   1,
   0,
   3,
@@ -191,7 +211,12 @@
   0,
   0,
   0,
-  2,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
   1,
   0,
   0,
@@ -226,7 +251,12 @@
   0,
   0,
   0,
-  2,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
   1,
   0,
   0,
@@ -255,6 +285,11 @@
   1,
   0,
   0,
+  0,
+  1,
+  1,
+  41,
+  0,
   1,
   0,
   3,
@@ -275,35 +310,255 @@
   4,
   2,
   0,
+  3,
+  0,
+  98,
+  41,
+  0,
+  1,
+  0,
+  60,
+  41,
+  0,
+  0,
+  60,
+  1,
+  0,
+  0,
+  0,
+  1,
+  22,
+  24,
+  0,
+  0,
+  22,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  41,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  25,
+  0,
+  1,
+  0,
+  15,
+  25,
+  0,
+  0,
+  15,
+  1,
+  0,
+  0,
+  0,
+  1,
+  7,
+  41,
+  0,
+  0,
+  7,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  24,
+  26,
+  0,
+  0,
+  24,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  11,
+  41,
+  0,
+  0,
+  11,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  3,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  3,
+  18,
+  0,
+  0,
+  3,
+  1,
+  0,
+  0,
+  0,
+  1,
+  19,
+  41,
+  0,
+  1,
+  0,
+  30,
+  41,
+  0,
   2,
   0,
   1,
   0,
   0,
   0,
+  2,
+  19,
+  27,
+  0,
+  0,
+  19,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  28,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  5,
+  27,
+  0,
+  0,
   5,
   1,
   0,
   0,
   0,
+  1,
+  6,
+  29,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
   2,
+  23,
+  31,
+  0,
+  0,
+  23,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  32,
+  0,
+  0,
+  4,
   1,
   0,
   0,
   0,
+  1,
   5,
+  31,
+  0,
+  0,
+  5,
+  1,
+  0,
+  0,
+  0,
+  1,
+  6,
+  33,
+  0,
+  0,
+  6,
   1,
   0,
   0,
   0,
+  1,
+  1,
+  31,
+  0,
   2,
+  0,
   1,
   0,
   0,
   0,
+  2,
+  17,
+  35,
+  0,
+  0,
+  17,
   1,
+  0,
+  0,
+  0,
   1,
+  4,
+  36,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
   0,
+  1,
+  5,
+  35,
   0,
   0,
   5,
@@ -312,11 +567,106 @@
   0,
   0,
   1,
+  6,
+  37,
+  0,
+  0,
+  6,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  35,
+  0,
+  1,
+  0,
   1,
   0,
   0,
   0,
   2,
+  13,
+  35,
+  0,
+  2,
+  0,
+  3,
+  0,
+  0,
+  2,
+  0,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  24,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  25,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  4,
+  26,
+  0,
+  0,
+  4,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  0,
+  0,
+  0,
+  1,
+  1,
+  41,
+  0,
+  0,
+  1,
   1,
   0,
   0,
@@ -395,36 +745,76 @@
   1,
   0,
   0,
+  0,
+  2,
+  4,
+  35,
+  0,
   1,
   0,
   1,
   0,
   0,
+  0,
+  2,
+  9,
+  35,
+  0,
   1,
   2,
   2,
   0,
   0,
+  0,
+  3,
+  11,
+  35,
+  0,
   1,
   2,
   2,
   0,
   0,
+  0,
+  3,
+  4,
+  35,
+  0,
   2,
   2,
   1,
   0,
   0,
+  0,
+  1,
+  14,
+  35,
+  0,
+  2,
   4,
+  11,
+  35,
+  0,
+  2,
   0,
   1,
   0,
   0,
+  0,
+  2,
+  8,
+  27,
+  0,
   2,
   0,
   2,
   0,
   0,
+  0,
+  3,
+  8,
+  27,
+  0,
   2,
   0,
   1,

From 6207638d50c366241b93bbfce44a26ced13c1096 Mon Sep 17 00:00:00 2001
From: David Thrane Christiansen 
Date: Thu, 7 May 2026 02:41:41 +0200
Subject: [PATCH 2/3] fix: account for upstream bugfix in test

---
 .../semanticTokensVersoDocs.lean.out.expected        | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected b/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
index c43b3924d404..278983311cd0 100644
--- a/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
+++ b/tests/server_interactive/semanticTokensVersoDocs.lean.out.expected
@@ -1,17 +1,7 @@
 {"version": 1,
  "uri": "file:///semanticTokensVersoDocs.lean",
  "isIncremental": false,
- "diagnostics":
- [{"source": "Lean 4",
-   "severity": 1,
-   "range":
-   {"start": {"line": 21, "character": 0},
-    "end": {"line": 21, "character": 45}},
-   "message":
-   "Unsupported syntax: failed to pretty print term (use 'set_option pp.rawOnError true' for raw representation)",
-   "fullRange":
-   {"start": {"line": 21, "character": 0},
-    "end": {"line": 21, "character": 45}}}]}
+ "diagnostics": []}
 {"textDocument": {"uri": "file:///semanticTokensVersoDocs.lean"},
  "position": {"line": 51, "character": 2}}
 {"data":

From af165e2fc28ce484d7bcbbdaa7b09dc37f46289d Mon Sep 17 00:00:00 2001
From: David Thrane Christiansen 
Date: Thu, 7 May 2026 03:04:50 +0200
Subject: [PATCH 3/3] fix: restructure test to make linter happy

---
 tests/markdown_conformance/run_test.sh                        | 4 ++--
 .../{conformance.out.expected => runner.lean.out.expected}    | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/markdown_conformance/{conformance.out.expected => runner.lean.out.expected} (100%)

diff --git a/tests/markdown_conformance/run_test.sh b/tests/markdown_conformance/run_test.sh
index dc02ac7bb319..204f7d91bfd7 100644
--- a/tests/markdown_conformance/run_test.sh
+++ b/tests/markdown_conformance/run_test.sh
@@ -1,9 +1,9 @@
 # With `-v`, every diverging example is printed to stdout, so a regression
 # turning a passing example into a failing one (or vice versa) shows up
-# example-level in the diff against `conformance.out.expected`, not merely
+# example-level in the diff against `runner.lean.out.expected`, not merely
 # as a per-section tally shift. The runner exits 0 when every non-skipped
 # example matches the spec.
-capture_only "conformance" \
+capture_only "runner.lean" \
   lean -Dlinter.all=false --run runner.lean -v spec.txt
 check_exit_is_success
 check_out_file
diff --git a/tests/markdown_conformance/conformance.out.expected b/tests/markdown_conformance/runner.lean.out.expected
similarity index 100%
rename from tests/markdown_conformance/conformance.out.expected
rename to tests/markdown_conformance/runner.lean.out.expected