Add CodSpeed continuous performance benchmarks

.github/workflows/codspeed.yml

@@ -0,0 +1,35 @@
+name: codspeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write # for OpenID Connect authentication with CodSpeed
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
+        with:
+          package-manager-cache: false
+          node-version: '24'
+      - run: npm i -g --force corepack && corepack enable
+      - run: pnpm install
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v5
+        with:
+          mode: simulation
+          run: pnpm bench

benchmarks/fixtures.ts

@@ -0,0 +1,150 @@
+// Shared markdown fixtures for the CodSpeed benchmark suite (`*.bench.ts`).
+// They are intentionally deterministic so results stay comparable across runs.
+
+/** A short document, close to a chat message or a README intro. */
+export const smallMarkdown = `# Hello World
+
+This is a **markdown** paragraph with *italic* text, \`inline code\` and a
+[link](https://example.com).
+
+- List item 1
+- List item 2
+- List item 3
+`
+
+/** A mid-sized document exercising most of the CommonMark + GFM surface. */
+export const mediumMarkdown = `---
+title: Benchmark Test
+description: A document covering the common Markdown surface
+tags:
+  - markdown
+  - benchmark
+---
+
+# Hello World
+
+This is a **markdown** document with *italic* text and [links](https://example.com).
+
+## Features
+
+- List item 1
+- List item 2 with \`inline code\`
+- List item 3
+
+### Code Block
+
+\`\`\`javascript
+const hello = 'world'
+console.log(hello)
+\`\`\`
+
+### Tables
+
+| Header 1 | Header 2 | Header 3 |
+|----------|----------|----------|
+| Cell 1   | Cell 2   | Cell 3   |
+| Cell 4   | Cell 5   | Cell 6   |
+
+### Components
+
+::alert{type="info"}
+This is an alert component with **bold** content.
+::
+
+::card{title="My Card"}
+Card content here
+::
+
+A [span]{.text-primary} and an image ![alt](https://example.com/img.png).
+
+### More Content
+
+1. Numbered list
+2. Another item
+3. Final item
+
+> This is a blockquote with some **bold** text
+
+- [ ] An open task
+- [x] A completed task
+
+~~Strikethrough text~~
+
+<div class="raw-html">
+  <span>Inline <b>HTML</b> block</span>
+</div>
+`
+
+/** A long document, representative of a full documentation page. */
+export const largeMarkdown = `---
+title: Large Benchmark Document
+---
+
+# API Reference
+
+${Array.from(
+  { length: 40 },
+  (_, i) => `
+## Module ${i + 1}
+
+Paragraph ${i + 1} with **bold**, *italic*, \`code\` and a [link](https://example.com/${i}).
+
+\`\`\`typescript
+export function module${i + 1}(input: string): string {
+  const result = input.trim()
+  return result.length > 0 ? result : 'default'
+}
+\`\`\`
+
+| Option | Type | Default |
+|--------|------|---------|
+| \`a\`  | \`string\` | \`'${i}'\` |
+| \`b\`  | \`number\` | \`${i}\` |
+
+::alert{type="info"}
+Note ${i + 1} about **module ${i + 1}**.
+::
+
+- item ${i}.1
+- item ${i}.2
+- item ${i}.3
+
+> Quote ${i + 1}
+`
+).join('\n')}
+`
+
+/**
+ * Truncated markdown, as produced by an LLM mid-stream: unclosed emphasis,
+ * an open code fence, a half written table and an unterminated component.
+ */
+export const partialMarkdown = `---
+title: Streaming
+---
+
+# Streaming output
+
+Here is some **bold text that is not
+
+\`\`\`typescript
+export function incomplete(input: string) {
+  const value = input
+
+| Header 1 | Header 2 |
+|----------|----------|
+| Cell 1   |
+
+::alert{type="info"}
+An alert that is still *being writt
+`
+
+/** Progressive chunks of `mediumMarkdown`, used to simulate a stream. */
+export const streamChunks: string[] = (() => {
+  const chunks: string[] = []
+  const size = Math.ceil(mediumMarkdown.length / 12)
+  for (let i = size; i < mediumMarkdown.length; i += size) {
+    chunks.push(mediumMarkdown.slice(0, i))
+  }
+  chunks.push(mediumMarkdown)
+  return chunks
+})()

benchmarks/parse.bench.ts

@@ -0,0 +1,37 @@
+import { bench, describe } from 'vitest'
+import { createMarkdownParser, parseMarkdown } from 'comark'
+import { largeMarkdown, mediumMarkdown, smallMarkdown } from './fixtures.ts'
+
+// Parsers are created once: `createMarkdownParser()` is the documented way to
+// reuse a configured parser, so the benchmarks measure parsing only.
+const parse = createMarkdownParser()
+const parseWithoutAutoClose = createMarkdownParser({ autoClose: false })
+const parseWithoutDefaultPlugins = createMarkdownParser({ registerDefaultPlugins: false })
+
+describe('parse', () => {
+  bench('small document', async () => {
+    await parse(smallMarkdown)
+  })
+
+  bench('medium document', async () => {
+    await parse(mediumMarkdown)
+  })
+
+  bench('large document', async () => {
+    await parse(largeMarkdown)
+  })
+})
+
+describe('parse options', () => {
+  bench('medium document without auto-close', async () => {
+    await parseWithoutAutoClose(mediumMarkdown)
+  })
+
+  bench('medium document without default plugins', async () => {
+    await parseWithoutDefaultPlugins(mediumMarkdown)
+  })
+
+  bench('medium document with a fresh parser instance', async () => {
+    await parseMarkdown(mediumMarkdown)
+  })
+})

benchmarks/plugins.bench.ts

@@ -0,0 +1,66 @@
+import { bench, describe } from 'vitest'
+import { createMarkdownParser } from 'comark'
+import emoji from 'comark/plugins/emoji'
+import footnotes from 'comark/plugins/footnotes'
+import punctuation from 'comark/plugins/punctuation'
+import security from 'comark/plugins/security'
+import summary from 'comark/plugins/summary'
+import toc from 'comark/plugins/toc'
+import { mediumMarkdown } from './fixtures.ts'
+
+const pluginMarkdown = `${mediumMarkdown}
+## Extras :rocket: :tada:
+
+"Smart quotes" -- with dashes... and a footnote[^1].
+
+[^1]: The footnote body with **bold** text.
+
+<script>alert('xss')</script>
+
+[Link](javascript:alert(1))
+`
+
+const baseline = createMarkdownParser()
+const withToc = createMarkdownParser({ plugins: [toc()] })
+const withEmoji = createMarkdownParser({ plugins: [emoji()] })
+const withPunctuation = createMarkdownParser({ plugins: [punctuation()] })
+const withFootnotes = createMarkdownParser({ plugins: [footnotes()] })
+const withSecurity = createMarkdownParser({ plugins: [security()] })
+const withSummary = createMarkdownParser({ plugins: [summary()] })
+const withAll = createMarkdownParser({
+  plugins: [toc(), emoji(), punctuation(), footnotes(), security(), summary()],
+})
+
+describe('plugins', () => {
+  bench('baseline (default plugins only)', async () => {
+    await baseline(pluginMarkdown)
+  })
+
+  bench('toc', async () => {
+    await withToc(pluginMarkdown)
+  })
+
+  bench('emoji', async () => {
+    await withEmoji(pluginMarkdown)
+  })
+
+  bench('punctuation', async () => {
+    await withPunctuation(pluginMarkdown)
+  })
+
+  bench('footnotes', async () => {
+    await withFootnotes(pluginMarkdown)
+  })
+
+  bench('security', async () => {
+    await withSecurity(pluginMarkdown)
+  })
+
+  bench('summary', async () => {
+    await withSummary(pluginMarkdown)
+  })
+
+  bench('all of the above combined', async () => {
+    await withAll(pluginMarkdown)
+  })
+})

benchmarks/render.bench.ts

@@ -0,0 +1,53 @@
+import { bench, describe } from 'vitest'
+import { createMarkdownParser } from 'comark'
+import { renderMarkdown } from 'comark/render'
+import { renderAnsiFromDocument } from '@comark/ansi'
+import { renderHtmlFromDocument } from '../packages/comark-html/src/index.ts'
+import { largeMarkdown, mediumMarkdown, smallMarkdown } from './fixtures.ts'
+
+const parse = createMarkdownParser()
+
+// Documents are parsed once so the benchmarks measure rendering only.
+const smallDocument = await parse(smallMarkdown)
+const mediumDocument = await parse(mediumMarkdown)
+const largeDocument = await parse(largeMarkdown)
+
+describe('render html', () => {
+  bench('small document', async () => {
+    await renderHtmlFromDocument(smallDocument)
+  })
+
+  bench('medium document', async () => {
+    await renderHtmlFromDocument(mediumDocument)
+  })
+
+  bench('large document', async () => {
+    await renderHtmlFromDocument(largeDocument)
+  })
+})
+
+describe('render ansi', () => {
+  bench('medium document', async () => {
+    await renderAnsiFromDocument(mediumDocument, { colors: true, width: 80 })
+  })
+
+  bench('large document', async () => {
+    await renderAnsiFromDocument(largeDocument, { colors: true, width: 80 })
+  })
+})
+
+describe('render markdown', () => {
+  bench('medium document', async () => {
+    await renderMarkdown(mediumDocument)
+  })
+
+  bench('large document', async () => {
+    await renderMarkdown(largeDocument)
+  })
+})
+
+describe('parse and render html', () => {
+  bench('medium document', async () => {
+    await renderHtmlFromDocument(await parse(mediumMarkdown))
+  })
+})

benchmarks/streaming.bench.ts

@@ -0,0 +1,38 @@
+import { bench, describe } from 'vitest'
+import { autoCloseMarkdown, createMarkdownParser } from 'comark'
+import { largeMarkdown, mediumMarkdown, partialMarkdown, streamChunks } from './fixtures.ts'
+
+const parse = createMarkdownParser()
+const streamingParse = createMarkdownParser()
+
+describe('auto-close', () => {
+  bench('partial document', () => {
+    autoCloseMarkdown(partialMarkdown, { frontmatter: true })
+  })
+
+  bench('medium document', () => {
+    autoCloseMarkdown(mediumMarkdown, { frontmatter: true })
+  })
+
+  bench('large document', () => {
+    autoCloseMarkdown(largeMarkdown, { frontmatter: true })
+  })
+})
+
+describe('streaming', () => {
+  bench('parse a partial document', async () => {
+    await parse(partialMarkdown)
+  })
+
+  bench('parse a partial document in streaming mode', async () => {
+    await streamingParse(partialMarkdown, { streaming: true })
+  })
+
+  // Re-parsing every growing chunk is what a renderer does while an LLM
+  // streams tokens, and it is the hot path of the incremental parser.
+  bench('parse a full stream of growing chunks', async () => {
+    for (const chunk of streamChunks) {
+      await streamingParse(chunk, { streaming: true })
+    }
+  })
+})