diff --git a/src/distill.js b/src/distill.js index 1eaa2a3..e40ceb6 100644 --- a/src/distill.js +++ b/src/distill.js @@ -191,7 +191,15 @@ function textToHTML(text, url = '', { full = false, type = '' } = {}) { const parts = paragraphs.map((p) => { const level = heading(p); if (level) return `${escHTML(p.slice(level + 1))}`; - return (full ? null : linkList(p, url)) ?? `
${escHTML(p)}
`; + if (full) return `
${escHTML(p)}
`; + // Expo's llms.txt writes each section heading on the line right above + // its list, with no blank line between, so the two arrive as one + // paragraph. The heading line is then a heading and the rest a list. + const cut = p.indexOf('\n'); + const top = cut > 0 ? heading(p.slice(0, cut)) : 0; + const list = linkList(top ? p.slice(cut + 1) : p, url); + if (!list) return `
${escHTML(p)}
`; + return top ? `${escHTML(p.slice(top + 1, cut))}${list}` : list; }); // Only a heading that opens the file names it. A README that starts with // prose and has a `# More information` section further down is not titled diff --git a/tests/text.test.js b/tests/text.test.js index 3799bfa..05b589f 100644 --- a/tests/text.test.js +++ b/tests/text.test.js @@ -74,6 +74,26 @@ test('the reference shortcuts resolve, and a file path keeps its slashes', () => assert.equal(resolveSite('gh', ['releases', 'facebook', 'react']).url, 'https://github.com/facebook/react/releases'); }); +test('a heading on the line right above its link list is a heading and a list', () => { + // Expo's llms.txt has no blank line between a section heading and its + // list, so the pair was one paragraph, which the heading line kept from + // being a list: every link in the section stayed verbatim text. + const text = '# Expo documentation\n\n### AI agents\n' + + '- [Claude Code and Expo](https://docs.expo.dev/agents/claude.md)\n' + + '- [Codex and Expo](https://docs.expo.dev/agents/codex.md)\n\n' + + '### Notes\nPlain prose under a heading.\n- [Not a list](https://docs.expo.dev/x.md)\n'; + const page = distill(text, 'https://docs.expo.dev/llms.txt'); + assert.deepEqual(page.blocks.filter((b) => b.href).map((b) => [b.text, b.href]), [ + ['Claude Code and Expo', 'https://docs.expo.dev/agents/claude.md'], + ['Codex and Expo', 'https://docs.expo.dev/agents/codex.md'], + ]); + const heading = page.blocks.find((b) => b.text === 'AI agents'); + assert.equal(heading?.type, 'heading', 'the heading line was lost or kept its #'); + // A heading above anything but a link list stays one verbatim block. + assert.ok(texts(page).some((l) => l.startsWith('### Notes\nPlain prose under a heading.'))); + assert.equal(toMarkdown(text, 'https://docs.expo.dev/llms.txt'), text.trimEnd(), 'raw rewrote the file'); +}); + // The shape of svelte.dev's llms.txt: a title, a summary, and sections of // links with a description each. const LLMS = '# Svelte Documentation for LLMs\n\n> Svelte is a UI framework.\n\n## Documentation Sets\n\n'