diff --git a/.changeset/cst-parser.md b/.changeset/cst-parser.md new file mode 100644 index 00000000000..516db5804a5 --- /dev/null +++ b/.changeset/cst-parser.md @@ -0,0 +1,5 @@ +--- +"@marko/compiler": patch +--- + +Drive the parser from the `@marko/parse` syntax tree instead of raw `htmljs-parser` events. diff --git a/packages/compiler/package.json b/packages/compiler/package.json index ba2771ed43e..0ba784dd59b 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -71,8 +71,8 @@ }, "dependencies": { "@luxass/strip-json-comments": "^2.0.1", + "@marko/parse": "link:../../../language-server/packages/parse", "complain": "^1.6.1", - "htmljs-parser": "^5.14.0", "jsesc": "^3.1.0", "kleur": "^4.1.5", "lasso-package-root": "^1.0.1", diff --git a/packages/compiler/src/babel-plugin/parser.js b/packages/compiler/src/babel-plugin/parser.js index f3a58ff70db..86c0b8e772e 100644 --- a/packages/compiler/src/babel-plugin/parser.js +++ b/packages/compiler/src/babel-plugin/parser.js @@ -10,7 +10,7 @@ import { parseVar, } from "@marko/compiler/babel-utils"; import { types as t } from "@marko/compiler/internal/babel"; -import { createParser, TagType } from "htmljs-parser"; +import { NodeType, parse, TagType } from "@marko/parse"; import { buildCodeFrameError } from "../util/build-code-frame"; import throwAggregateError from "../util/merge-errors"; @@ -50,17 +50,37 @@ export function parseMarko(file) { const { htmlParseOptions = {} } = file.markoOpts; const { watchFiles } = file.metadata.marko; const parseVisits = []; + // Tag definitions looked up by the parse hook, keyed by tag name start. + const tagDefs = new Map(); let currentTag = file.path; let currentBody = currentTag; - let currentAttr = undefined; - let currentShorthandId = undefined; - let currentShorthandClassNames = undefined; - let { preserveWhitespace } = htmlParseOptions; - let preservingWhitespaceUntil = preserveWhitespace; - let onNext = noop; - const positionAt = (index) => toBabelPosition(parser.positionAt(index)); + let preservingWhitespaceUntil = htmlParseOptions.preserveWhitespace; + // Trailing whitespace of a text node depends on what follows it, so its + // final value resolves once the next sibling (or end of body) is known. + let resolveText = noop; + + const parsed = parse(code, file.opts.filename, { + // The taglib decides how every tag body parses, so the parser's own + // default for the name is ignored. + getTagType(name, range) { + const tagDef = getTagDefForTagName(file, name); + const parseOptions = tagDef?.parseOptions; + tagDefs.set(range.start, tagDef); + + if (parseOptions) { + if (parseOptions.statement) return TagType.statement; + if (parseOptions.openTagOnly) return TagType.void; + if (parseOptions.text) return TagType.text; + } + + return TagType.html; + }, + }); + + const read = (range) => parsed.read(range); + const positionAt = (index) => toBabelPosition(parsed.positionAt(index)); const locationAt = (range) => { - const { start, end } = parser.locationAt(range); + const { start, end } = parsed.locationAt(range); return { start: toBabelPosition(start), end: toBabelPosition(end), @@ -99,30 +119,24 @@ export function parseMarko(file) { currentTag = currentBody.pushContainer("body", node)[0]; } currentBody = currentTag.get("body"); - onNext(node); + resolveText(node); }; const pushContent = (node) => { currentBody.node.body.push(node); - onNext(node); - }; - const endAttr = () => { - if (currentAttr) { - currentAttr.loc = locationAt(currentAttr); - currentAttr = undefined; - } + resolveText(node); }; const parseTemplateString = ({ quasis, expressions }) => { switch (expressions.length) { case 0: { const [first] = quasis; - return withLoc(t.stringLiteral(parser.read(first)), first); + return withLoc(t.stringLiteral(read(first)), first); } case 1: { if (emptyRange(quasis[0]) && emptyRange(quasis[1])) { const [{ value }] = expressions; const result = parseExpression( file, - parser.read(value), + read(value), value.start, value.end, ); @@ -141,567 +155,672 @@ export function parseMarko(file) { const [{ start }] = quasis; const end = quasis[quasis.length - 1].end; - return parseTemplateLiteral(file, parser.read({ start, end }), start, end); + return parseTemplateLiteral(file, read({ start, end }), start, end); }; - const parser = createParser({ - onError(part) { - const err = buildCodeFrameError( - file.opts.filename, - file.code, - locationAt(part), - part.message, - ); + const reportError = (part) => { + const err = buildCodeFrameError( + file.opts.filename, + file.code, + locationAt(part), + part.message, + ); - if (!file.___hasParseErrors) { - throw err; + if (!file.___hasParseErrors) { + throw err; + } + + const errors = []; + t.traverseFast(file.path.node, (node) => { + if (node.type === "MarkoParseError") { + errors.push( + buildCodeFrameError( + file.opts.filename, + file.code, + node.errorLoc || node.loc, + node.label, + ), + ); } + }); - const errors = []; - t.traverseFast(file.path.node, (node) => { - if (node.type === "MarkoParseError") { - errors.push( - buildCodeFrameError( - file.opts.filename, - file.code, - node.errorLoc || node.loc, - node.label, + errors.push(err); + throwAggregateError(errors); + }; + + const visitBody = (body) => { + for (const child of body) visitNode(child); + }; + + const visitNode = (node) => { + switch (node.type) { + case NodeType.Text: + visitText(node); + break; + case NodeType.Placeholder: + pushContent( + withLoc( + t.markoPlaceholder( + parseExpression( + file, + read(node.value), + node.value.start, + node.value.end, + ), + node.escape, ), - ); - } - }); + node, + ), + ); + break; + case NodeType.Scriptlet: + pushContent( + withLoc( + t.markoScriptlet( + parseStatements( + file, + read(node.value), + node.value.start, + node.value.end, + ), + ), + node, + ), + ); + break; + case NodeType.CDATA: + pushContent(withLoc(t.markoCDATA(read(node.value)), node)); + break; + case NodeType.Doctype: + pushContent(withLoc(t.markoDocumentType(read(node.value)), node)); + break; + case NodeType.Declaration: + pushContent(withLoc(t.markoDeclaration(read(node.value)), node)); + break; + case NodeType.Comment: + pushContent(withLoc(t.markoComment(read(node.value)), node)); + break; + case NodeType.Import: + case NodeType.Export: + case NodeType.Class: + case NodeType.Static: + visitStatic(node); + break; + case NodeType.Style: + visitStyle(node); + break; + case NodeType.Tag: + case NodeType.AttrTag: + visitTag(node); + break; + } + }; - errors.push(err); - throwAggregateError(errors); - }, - onText(part) { - const rawValue = parser.read(part); + const visitText = (part) => { + const rawValue = read(part); - if (preservingWhitespaceUntil) { - pushContent(withLoc(t.markoText(rawValue), part)); - return; - } + if (preservingWhitespaceUntil) { + pushContent(withLoc(t.markoText(rawValue), part)); + return; + } - if (/^(?:[\n\r]\s*)?(?:[\n\r]\s*)?$/.test(rawValue)) return; + if (/^(?:[\n\r]\s*)?(?:[\n\r]\s*)?$/.test(rawValue)) return; - const { body } = currentBody.node; - let prev; - let prevIndex = body.length; - // Find previous non-scriptlet or comment. - while (prevIndex > 0) { - prev = body[--prevIndex]; + const { body } = currentBody.node; + let prev; + let prevIndex = body.length; + // Find previous non-scriptlet or comment. + while (prevIndex > 0) { + prev = body[--prevIndex]; - if (t.isMarkoScriptlet(prev) || t.isMarkoComment(prev)) { - prev = undefined; - } else { - break; - } + if (t.isMarkoScriptlet(prev) || t.isMarkoComment(prev)) { + prev = undefined; + } else { + break; } + } + + let value = rawValue; + switch (prev?.type) { + case "MarkoPlaceholder": + break; + case "MarkoText": + if (/\s$/.test(prev.value)) { + value = value.replace(/^\s+/, ""); + } + break; + case "MarkoTag": + if (isStatementTag(prev) || isAttrTag(prev)) { + value = value.replace(/^[\n\r]\s*/, ""); + } + break; + default: + value = value.replace(/^[\n\r]\s*/, ""); + break; + } + + if (!value) return; - let value = rawValue; - switch (prev?.type) { + const node = t.markoText(value); + pushContent(node); + resolveText = (next) => { + switch (next?.type) { + case "MarkoScriptlet": + case "MarkoComment": + return; case "MarkoPlaceholder": break; case "MarkoText": - if (/\s$/.test(prev.value)) { - value = value.replace(/^\s+/, ""); + if (/^\s/.test(next.value)) { + value = value.replace(/\s+$/, ""); } break; case "MarkoTag": - if (isStatementTag(prev) || isAttrTag(prev)) { - value = value.replace(/^[\n\r]\s*/, ""); + if (isStatementTag(next) || isAttrTag(next)) { + value = value.replace(/[\n\r]\s*$/, ""); } + break; default: - value = value.replace(/^[\n\r]\s*/, ""); + value = value.replace(/[\n\r]\s*$/, ""); break; } - if (!value) return; - - const node = t.markoText(value); - pushContent(node); - onNext = (next) => { - switch (next?.type) { - case "MarkoScriptlet": - case "MarkoComment": - return; - case "MarkoPlaceholder": - break; - case "MarkoText": - if (/^\s/.test(next.value)) { - value = value.replace(/\s+$/, ""); - } - break; - case "MarkoTag": - if (isStatementTag(next) || isAttrTag(next)) { - value = value.replace(/[\n\r]\s*$/, ""); - } - - break; - default: - value = value.replace(/[\n\r]\s*$/, ""); - break; - } - - node.value = value.replace(/\s+/g, " "); + node.value = value.replace(/\s+/g, " "); - if (node.value) { - const trimmedStart = part.start + rawValue.indexOf(value); - withLoc(node, { - start: trimmedStart, - end: trimmedStart + value.length, - }); - } else { - body.splice(body.indexOf(node), 1); - } + if (node.value) { + const trimmedStart = part.start + rawValue.indexOf(value); + withLoc(node, { + start: trimmedStart, + end: trimmedStart + value.length, + }); + } else { + body.splice(body.indexOf(node), 1); + } - onNext = noop; - }; - }, - onCDATA(part) { - pushContent(withLoc(t.markoCDATA(parser.read(part.value)), part)); - }, - onDoctype(part) { - pushContent(withLoc(t.markoDocumentType(parser.read(part.value)), part)); - }, - onDeclaration(part) { - pushContent(withLoc(t.markoDeclaration(parser.read(part.value)), part)); - }, - onComment(part) { - pushContent(withLoc(t.markoComment(parser.read(part.value)), part)); - }, - onTagTypeArgs(part) { - currentTag.node.typeArguments = parseTypeArgs( - file, - parser.read(part.value), - part.value.start, - part.value.end, - ); - }, - onTagTypeParams(part) { - currentBody.node.typeParameters = parseTypeParams( - file, - parser.read(part.value), - part.value.start, - part.value.end, - ); - }, - onPlaceholder(part) { - pushContent( - withLoc( - t.markoPlaceholder( - parseExpression( - file, - parser.read(part.value), - part.value.start, - part.value.end, - ), - part.escape, - ), - part, - ), - ); - }, - onScriptlet(part) { - pushContent( - withLoc( - t.markoScriptlet( - parseStatements( - file, - parser.read(part.value), - part.value.start, - part.value.end, - ), - ), - part, - ), - ); - }, - onOpenTagName(part) { - const tagName = parseTemplateString(part); - const node = t.markoTag(tagName, [], t.markoTagBody()); - let parseType = TagType.html; - node.start = - part.start - (part.start && code[part.start - 1] === "<" ? 1 : 0); // Account for leading `<` in html mode. - node.end = part.end; - - if (t.isStringLiteral(tagName)) { - const literalTagName = tagName.value || (tagName.value = "div"); - - if (literalTagName === "%") { - throw file.buildCodeFrameError( - tagName, - "<% scriptlets %> are no longer supported.", - ); - } + resolveText = noop; + }; + }; - const parseOptions = (node.tagDef = getTagDefForTagName( - file, - literalTagName, - ))?.parseOptions; - - if (parseOptions) { - if (parseOptions.preserveWhitespace) { - // Keep the outermost owner so a nested preserving tag (eg a - // `