Update theme-check-common, prettier-plugin-liquid, and theme-graph for new parser

.changeset/hand-written-liquid-html-parser.md

@@ -0,0 +1,16 @@
+---
+'@shopify/liquid-html-parser': minor
+'@shopify/theme-language-server-common': minor
+'@shopify/theme-check-common': minor
+'@shopify/prettier-plugin-liquid': minor
+'@shopify/theme-graph': minor
+---
+
+Adopt the hand-written `liquid-html-parser` for performance
+
+Replace the parser-combinator-based Liquid/HTML parser with a hand-written
+recursive-descent parser. The new parser is significantly faster, adds
+resilient parsing (it recovers from malformed input instead of bailing), and is
+adapted to the theme-tools source model. `theme-language-server-common`,
+`theme-check-common`, `prettier-plugin-liquid`, and `theme-graph` are updated to
+consume the new parser.

packages/liquid-html-parser/src/ast.test.ts

@@ -1345,6 +1345,26 @@ describe('Unit: Stage 2 (AST)', () => {
       expect(useChild.type).to.eql('TextNode');
     });
 
+    it('should depth-balance nested same-name raw tags and close at the outer tag (issue 156)', () => {
+      const expectPath = makeExpectPath('toLiquidHtmlAST - nested raw close balancing');
+
+      // Nested <svg> elements: the outer <svg> must close at the LAST
+      // </svg>, keeping the inner <svg>…</svg> as raw body text rather than
+      // closing early at the first </svg> (which previously threw / mis-parsed).
+      ast = toLiquidHtmlAST('<svg>a<svg>b</svg>c</svg>');
+      expectPath(ast, 'children').to.have.lengthOf(1);
+      expectPath(ast, 'children.0.type').to.eql('HtmlRawNode');
+      expectPath(ast, 'children.0.name').to.eql('svg');
+      expectPath(ast, 'children.0.body.value').to.eql('a<svg>b</svg>c');
+
+      // Same balancing for a non-svg raw tag (<script>).
+      ast = toLiquidHtmlAST('<script>a<script>b</script>c</script>');
+      expectPath(ast, 'children').to.have.lengthOf(1);
+      expectPath(ast, 'children.0.type').to.eql('HtmlRawNode');
+      expectPath(ast, 'children.0.name').to.eql('script');
+      expectPath(ast, 'children.0.body.value').to.eql('a<script>b</script>c');
+    });
+
     it(`should parse a basic text node into a TextNode`, () => {
       for (const { toAST, expectPath, expectPosition } of testCases) {
         ast = toAST('Hello world!');

packages/liquid-html-parser/src/document/base.ts

@@ -54,6 +54,17 @@ export class ParserBase {
     return this.source;
   }
 
+  /**
+   * Whether this document parse is tolerant. The strict/default parser returns
+   * false; `TolerantDocumentParser` overrides it to true. Markup-construction
+   * sites consult this to enable the markup parser's tolerant recovery axis
+   * (`enableTolerant()`), leaving strict (and theme-check) parses untouched.
+   * Disjoint from the render-tree lax axis.
+   */
+  isTolerant(): boolean {
+    return false;
+  }
+
   tokenAt(index: number): Token {
     return this.tokens[index];
   }

packages/liquid-html-parser/src/document/factories.ts

@@ -105,12 +105,15 @@ export function makeLiquidTagBaseCase(
   blockEndPosition?: Position,
   delimiterWhitespace?: { start: LiquidOpenWhitespace; end: LiquidCloseWhitespace },
   reason?: string,
+  // `#`-comment lines in `{% liquid %}` keep their inner indentation verbatim;
+  // every other base-case tag trims (the default).
+  preserveMarkup: boolean = false,
 ): LiquidTagBaseCase {
   const posEnd = blockEndPosition ? blockEndPosition.end : envelope.blockStartPosition.end;
   return {
     type: NodeTypes.LiquidTag,
     name: envelope.tagName,
-    markup: envelope.markupString.trim(),
+    markup: preserveMarkup ? envelope.markupString : envelope.markupString.trim(),
     children,
     whitespaceStart: envelope.whitespaceStart,
     whitespaceEnd: envelope.whitespaceEnd,

packages/liquid-html-parser/src/document/html.ts

@@ -217,11 +217,15 @@ export function parseHtmlComment(parser: HtmlParserDelegate): HtmlComment {
   const openToken = parser.consume(TokenType.HtmlCommentOpen);
   const bodyStart = openToken.end;
 
-  while (!parser.isAtEnd() && !parser.check(TokenType.HtmlCommentClose)) {
-    parser.advance();
-  }
-
-  if (parser.isAtEnd()) {
+  // Find the closing `-->` by scanning the source rather than walking tokens.
+  // A conditional comment body (`<!--[if IE]>…<![endif]-->`) contains
+  // `<![endif]`, which the tokenizer treats as a doctype open — it enters
+  // HtmlTag mode and consumes the trailing `-->` as an HtmlTagClose, so no
+  // HtmlCommentClose token is ever emitted and the token walk would run to EOF.
+  // A source scan pins us to the real comment close, so the (unchanged)
+  // HtmlComment node feeds getConditionalComment correctly in the printer.
+  const closeIdx = source.indexOf('-->', bodyStart);
+  if (closeIdx === -1) {
     throw new LiquidHTMLASTParsingError(
       `Attempting to end parsing before HtmlComment '<!--' was closed`,
       source,
@@ -230,11 +234,13 @@ export function parseHtmlComment(parser: HtmlParserDelegate): HtmlComment {
     );
   }
 
-  const bodyEnd = parser.peek().start;
-  const closeToken = parser.consume(TokenType.HtmlCommentClose);
+  const bodyEnd = closeIdx;
+  const commentEnd = closeIdx + 3; // past `-->`
   const body = source.slice(bodyStart, bodyEnd).trim();
 
-  return makeHtmlComment(body, openToken.start, closeToken.end, source);
+  parser.seekToSourceOffset(commentEnd);
+
+  return makeHtmlComment(body, openToken.start, commentEnd, source);
 }
 
 // htmlDoctype := "<!" text ">"
@@ -499,7 +505,11 @@ function parseAttributeList(parser: HtmlParserDelegate): AttributeNode[] {
         const attrEnd = closeQuote.end;
         const attributePosition: Position = { start: valueStart, end: valueEnd };
 
-        if (quoteChar === '"') {
+        // Double straight quote and double curly quotes (“ ”) map to a
+        // double-quoted attr; single straight quote and single curly quotes
+        // (‘ ’) map to a single-quoted attr. The printer normalizes the curly
+        // variants to straight quotes.
+        if (quoteChar === '"' || quoteChar === '“' || quoteChar === '”') {
           attrs.push(
             makeAttrDoubleQuoted(name, value, attributePosition, attrStart, attrEnd, source),
           );
@@ -738,13 +748,39 @@ export function scanForHtmlCloseTag(parser: ParserBase, tagName: string): number
   const tokenCount = parser.tokenCount();
   const pos = parser.getPosition();
 
+  // Depth-balance nested same-name elements so the OUTER close tag is
+  // returned, not the first inner one (e.g. `<svg>…<svg>…</svg>…</svg>`). A
+  // nested open tag is an `HtmlTagOpen` followed by a `Text` token whose first
+  // word matches the tag name — the tokenizer folds the tag name and any
+  // trailing attributes into a single text token, so we take the first word.
+  // The scan begins after the outer open tag has been consumed, so the outer
+  // open is never counted. Mirrors `scanForEndTagNested`.
+  let depth = 0;
   for (let i = pos; i < tokenCount; i++) {
-    if (parser.tokenAt(i).type !== TokenType.HtmlCloseTagOpen) continue;
+    const token = parser.tokenAt(i);
+
+    if (token.type === TokenType.HtmlTagOpen) {
+      const textIdx = i + 1;
+      if (textIdx >= tokenCount) continue;
+      if (parser.tokenAt(textIdx).type !== TokenType.Text) continue;
+      const trimmed = source
+        .slice(parser.tokenAt(textIdx).start, parser.tokenAt(textIdx).end)
+        .trimStart();
+      const firstWs = trimmed.search(/\s/);
+      const name = firstWs === -1 ? trimmed.trim() : trimmed.slice(0, firstWs);
+      if (name.toLowerCase() === lowerName) depth++;
+      continue;
+    }
+
+    if (token.type !== TokenType.HtmlCloseTagOpen) continue;
     const textIdx = i + 1;
     if (textIdx >= tokenCount) continue;
     if (parser.tokenAt(textIdx).type !== TokenType.Text) continue;
     const text = source.slice(parser.tokenAt(textIdx).start, parser.tokenAt(textIdx).end);
-    if (text.trim().toLowerCase() === lowerName) return i;
+    if (text.trim().toLowerCase() === lowerName) {
+      if (depth === 0) return i;
+      depth--;
+    }
   }
   return -1;
 }
@@ -768,6 +804,7 @@ function scriptKindFromAttributes(attributes: AttributeNode[]): RawMarkupKinds {
   const typeValue = extractPlainAttributeValue(attributes, 'type');
   if (typeValue === null) return RawMarkupKinds.javascript;
   if (typeValue === 'text/html') return RawMarkupKinds.html;
+  if (typeValue === 'text/markdown') return RawMarkupKinds.markdown;
   if (
     typeValue.endsWith('json') ||
     typeValue.endsWith('importmap') ||

packages/liquid-html-parser/src/document/liquid-blocks.ts

@@ -68,6 +68,7 @@ export function parseBlockTag(
     const markupStringStart = closeToken.start - envelope.markupString.length;
     const tokens = tokenizeMarkup(envelope.markupString, markupStringStart);
     const markupParser = new MarkupParser(tokens, parser.getSource());
+    if (parser.isTolerant()) markupParser.enableTolerant();
     markup = def.parse(envelope.tagName, markupParser, parser);
     if (!markupParser.isAtEnd()) {
       markup = undefined;
@@ -341,6 +342,7 @@ export function parseBranchMarkup(
       try {
         const tokens = tokenizeMarkup(envelope.markupString, markupStringStart);
         const markupParser = new MarkupParser(tokens, parser.getSource());
+        if (parser.isTolerant()) markupParser.enableTolerant();
         const result = elsifBranchParse(branchName, markupParser);
         if (!markupParser.isAtEnd()) return envelope.markupString.trim();
         return result;
@@ -352,6 +354,7 @@ export function parseBranchMarkup(
       try {
         const tokens = tokenizeMarkup(envelope.markupString, markupStringStart);
         const markupParser = new MarkupParser(tokens, parser.getSource());
+        if (parser.isTolerant()) markupParser.enableTolerant();
         const result = whenBranchParse(branchName, markupParser);
         if (!markupParser.isAtEnd()) return envelope.markupString.trim();
         return result;

packages/liquid-html-parser/src/document/liquid-hybrid.ts

@@ -31,6 +31,7 @@ export function parseHybridTag(
     const markupStringStart = closeToken.start - envelope.markupString.length;
     const tokens = tokenizeMarkup(envelope.markupString, markupStringStart);
     const markupParser = new MarkupParser(tokens, parser.getSource());
+    if (parser.isTolerant()) markupParser.enableTolerant();
     markup = def.parse(envelope.tagName, markupParser, parser);
     if (!markupParser.isAtEnd()) {
       markup = undefined;

packages/liquid-html-parser/src/document/liquid-lines.test.ts

@@ -112,6 +112,14 @@ describe('Unit: liquid-lines', () => {
     expectPath(ast, 'children.0.markup.1.name').to.eql('echo');
   });
 
+  it('should preserve inner indentation after `#`, stripping only one separator space (Category J)', () => {
+    // `#     fancy` must keep four spaces of the ASCII-art indent — only a
+    // single separator space after `#` is stripped (ohm `"#" space?`).
+    const ast = toLiquidHtmlAST('{% liquid\n#     fancy\necho "hi"\n%}');
+    expectPath(ast, 'children.0.markup.0.name').to.eql('#');
+    expectPath(ast, 'children.0.markup.0.markup').to.eql('    fancy');
+  });
+
   it('should skip empty lines', () => {
     const ast = toLiquidHtmlAST('{% liquid\n\necho "hi"\n\n%}');
     expectPath(ast, 'children.0.markup').to.have.lengthOf(1);
@@ -226,4 +234,33 @@ describe('Unit: liquid-lines', () => {
       expectPath(ast, `${branch}.blockEndPosition.end`).to.eql(source.indexOf('endif'));
     });
   });
+
+  describe('nested comment/doc balancing (Category B)', () => {
+    it('should balance nested comments so the outer endcomment closes the block', () => {
+      // The inner `comment`/`endcomment` pair must not close the outer
+      // block early; the outer comment ends at the final `endcomment`.
+      const source = '{% liquid\ncomment\nouter\ncomment\ninner\nendcomment\nendcomment\n%}';
+      const ast = toLiquidHtmlAST(source);
+      expectPath(ast, 'children.0.markup').to.have.lengthOf(1);
+      expectPath(ast, 'children.0.markup.0.type').to.eql('LiquidRawTag');
+      expectPath(ast, 'children.0.markup.0.name').to.eql('comment');
+      expectPath(ast, 'children.0.markup.0.body.value').to.eql(
+        'outer\ncomment\ninner\nendcomment\n',
+      );
+    });
+
+    it('should carve out a nested raw so a literal endcomment inside it does not close early', () => {
+      // The `endcomment` line sits inside a nested `raw`…`endraw` block, so it
+      // must be ignored by the depth scan; the outer comment closes at the
+      // real trailing `endcomment`.
+      const source = '{% liquid\ncomment\nbefore\nraw\nendcomment\nendraw\nafter\nendcomment\n%}';
+      const ast = toLiquidHtmlAST(source);
+      expectPath(ast, 'children.0.markup').to.have.lengthOf(1);
+      expectPath(ast, 'children.0.markup.0.type').to.eql('LiquidRawTag');
+      expectPath(ast, 'children.0.markup.0.name').to.eql('comment');
+      expectPath(ast, 'children.0.markup.0.body.value').to.eql(
+        'before\nraw\nendcomment\nendraw\nafter\n',
+      );
+    });
+  });
 });

packages/liquid-html-parser/src/document/liquid-lines.ts

@@ -64,7 +64,9 @@ export function parseLiquidStatement(
   const envelope = envelopeFromLine(line, parser.getSource());
 
   if (tagName === '#') {
-    return makeLiquidTagBaseCase(envelope);
+    // Preserve the inline comment's inner indentation (`preserveMarkup`); only
+    // the single separator space after `#` was stripped upstream.
+    return makeLiquidTagBaseCase(envelope, undefined, undefined, undefined, undefined, true);
   }
 
   if (tagName.startsWith('end')) {
@@ -93,6 +95,7 @@ export function parseLiquidStatement(
           markupOffset,
           envelope.markupEnd,
         );
+        if (parser.isTolerant()) markupParser.enableTolerant();
 
         const markup = def.parse(tagName, markupParser, parser);
         if (!markupParser.isAtEnd()) {
@@ -140,6 +143,7 @@ export function parseLineBlockTag(
   try {
     const tokens = tokenizeMarkup(markupString, markupOffset);
     const markupParser = new MarkupParser(tokens, parser.getSource());
+    if (parser.isTolerant()) markupParser.enableTolerant();
 
     markup = def.parse(envelope.tagName, markupParser, parser);
     if (!markupParser.isAtEnd()) {
@@ -196,11 +200,41 @@ export function parseLineRawTag(
       ? ctx.lines[ctx.index - 1].lineEnd + 1
       : envelope.blockStartPosition.end;
 
+  // `comment`/`doc` bodies balance nested opens of the same tag before
+  // matching their end line, mirroring the document-path scan in
+  // liquid-raw.ts (Ruby comment.rb v5.13.0 `comment_tag_depth`). A nested
+  // `raw` block is carved out: `raw` is first-match and does not nest, so an
+  // `endcomment`/`comment` word sitting on a line inside a raw body must not
+  // affect the depth. Every other raw tag (`raw`, `javascript`, `schema`,
+  // `style`) keeps the original first-match scan and stays byte-identical.
+  const balanced = tagName === 'comment' || tagName === 'doc';
   let endLineIndex = -1;
-  for (let i = ctx.index; i < ctx.lines.length; i++) {
-    if (ctx.lines[i].tagName === endTagName) {
-      endLineIndex = i;
-      break;
+  if (balanced) {
+    let depth = 0;
+    for (let i = ctx.index; i < ctx.lines.length; i++) {
+      const name = ctx.lines[i].tagName;
+      if (name === 'raw') {
+        // Skip past the nested raw block; its body lines never affect depth.
+        i++;
+        while (i < ctx.lines.length && ctx.lines[i].tagName !== 'endraw') i++;
+        continue;
+      }
+      if (name === tagName) {
+        depth++;
+      } else if (name === endTagName) {
+        if (depth === 0) {
+          endLineIndex = i;
+          break;
+        }
+        depth--;
+      }
+    }
+  } else {
+    for (let i = ctx.index; i < ctx.lines.length; i++) {
+      if (ctx.lines[i].tagName === endTagName) {
+        endLineIndex = i;
+        break;
+      }
     }
   }
 
@@ -262,6 +296,7 @@ export function parseLineHybridTag(
   try {
     const tokens = tokenizeMarkup(markupString, markupOffset);
     const markupParser = new MarkupParser(tokens, parser.getSource());
+    if (parser.isTolerant()) markupParser.enableTolerant();
 
     markup = def.parse(envelope.tagName, markupParser, parser);
     if (!markupParser.isAtEnd()) {
@@ -385,6 +420,7 @@ export function parseLineBranchedBody(
         line.tagName as BranchName,
         branchEnvelope,
         parser.getSource(),
+        parser.isTolerant(),
       );
       currentBranch = makeLiquidBranchNamed(branchEnvelope, branchMarkup);
       currentChildren = [];
@@ -423,12 +459,14 @@ export function parseLineBranchMarkup(
   branchName: BranchName,
   envelope: LiquidTagEnvelope,
   source: string,
+  tolerant: boolean = false,
 ): unknown {
   switch (branchName) {
     case 'elsif': {
       try {
         const tokens = tokenizeMarkup(envelope.markupString, envelope.markupOffset);
         const markupParser = new MarkupParser(tokens, source);
+        if (tolerant) markupParser.enableTolerant();
 
         const result = elsifBranchParse(branchName, markupParser);
         if (!markupParser.isAtEnd()) return envelope.markupString.trim();
@@ -441,6 +479,7 @@ export function parseLineBranchMarkup(
       try {
         const tokens = tokenizeMarkup(envelope.markupString, envelope.markupOffset);
         const markupParser = new MarkupParser(tokens, source);
+        if (tolerant) markupParser.enableTolerant();
 
         const result = whenBranchParse(branchName, markupParser);
         if (!markupParser.isAtEnd()) return envelope.markupString.trim();