Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/distill.js
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,15 @@ function textToHTML(text, url = '', { full = false, type = '' } = {}) {
const parts = paragraphs.map((p) => {
const level = heading(p);
if (level) return `<h${level}>${escHTML(p.slice(level + 1))}</h${level}>`;
return (full ? null : linkList(p, url)) ?? `<pre>${escHTML(p)}</pre>`;
if (full) return `<pre>${escHTML(p)}</pre>`;
// Expo's llms.txt writes each section heading on the line right above
// its list, with no blank line between, so the two arrive as one
// paragraph. The heading line is then a heading and the rest a list.
const cut = p.indexOf('\n');
const top = cut > 0 ? heading(p.slice(0, cut)) : 0;
const list = linkList(top ? p.slice(cut + 1) : p, url);
if (!list) return `<pre>${escHTML(p)}</pre>`;
return top ? `<h${top}>${escHTML(p.slice(top + 1, cut))}</h${top}>${list}` : list;
});
// Only a heading that opens the file names it. A README that starts with
// prose and has a `# More information` section further down is not titled
Expand Down
20 changes: 20 additions & 0 deletions tests/text.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,26 @@ test('the reference shortcuts resolve, and a file path keeps its slashes', () =>
assert.equal(resolveSite('gh', ['releases', 'facebook', 'react']).url, 'https://github.com/facebook/react/releases');
});

test('a heading on the line right above its link list is a heading and a list', () => {
// Expo's llms.txt has no blank line between a section heading and its
// list, so the pair was one paragraph, which the heading line kept from
// being a list: every link in the section stayed verbatim text.
const text = '# Expo documentation\n\n### AI agents\n'
+ '- [Claude Code and Expo](https://docs.expo.dev/agents/claude.md)\n'
+ '- [Codex and Expo](https://docs.expo.dev/agents/codex.md)\n\n'
+ '### Notes\nPlain prose under a heading.\n- [Not a list](https://docs.expo.dev/x.md)\n';
const page = distill(text, 'https://docs.expo.dev/llms.txt');
assert.deepEqual(page.blocks.filter((b) => b.href).map((b) => [b.text, b.href]), [
['Claude Code and Expo', 'https://docs.expo.dev/agents/claude.md'],
['Codex and Expo', 'https://docs.expo.dev/agents/codex.md'],
]);
const heading = page.blocks.find((b) => b.text === 'AI agents');
assert.equal(heading?.type, 'heading', 'the heading line was lost or kept its #');
// A heading above anything but a link list stays one verbatim block.
assert.ok(texts(page).some((l) => l.startsWith('### Notes\nPlain prose under a heading.')));
assert.equal(toMarkdown(text, 'https://docs.expo.dev/llms.txt'), text.trimEnd(), 'raw rewrote the file');
});

// The shape of svelte.dev's llms.txt: a title, a summary, and sections of
// links with a description each.
const LLMS = '# Svelte Documentation for LLMs\n\n> Svelte is a UI framework.\n\n## Documentation Sets\n\n'
Expand Down
Loading