fix(editor): html parsing support for unquoted attributes

__tests__/lib/mdxish/utils/mdxish-component-tag-parser.test.ts

@@ -1,8 +1,11 @@
 import type { MdxJsxAttributeValueExpression } from 'mdast-util-mdx-jsx';
 
+import * as rmdx from '@readme/markdown-legacy';
+
 import { mdxish } from '../../../../lib';
 import { parseAttributes, parseTag } from '../../../../lib/utils/mdxish/mdxish-component-tag-parser';
-import { parseMdxishWithSource } from '../../../helpers';
+import { extractText } from '../../../../processor/transform/extract-text';
+import { findElementByTagName, parseMdxishWithSource } from '../../../helpers';
 
 describe('parseAttributes', () => {
   describe('boolean attributes', () => {
@@ -452,4 +455,60 @@ describe('lowercase html tags with JSX expressions are treated as MDX', () => {
       }],
     });
   });
+
+  describe.each([
+    ['MDXish', mdxish],
+    ['legacy RMDX', rmdx.hast],
+  ])('RM-16375 unquoted native HTML attributes in %s', (_engine, parse) => {
+    it.each([
+      'Pass <key=value> to the CLI.',
+      'Compare <a = b> and <c = d>.',
+    ])('should preserve tag-like prose: %s', source => {
+      expect(extractText(parse(source))).toBe(source);
+    });
+
+    it('should preserve paragraph structure inside a block tag', () => {
+      const tree = parse('<div id=a>\n\npara one\n\npara two\n\n</div>');
+      const div = findElementByTagName(tree, 'div');
+
+      expect(div?.children.filter(child => child.type === 'element')).toMatchObject([
+        { type: 'element', tagName: 'p', children: [{ type: 'text', value: 'para one' }] },
+        { type: 'element', tagName: 'p', children: [{ type: 'text', value: 'para two' }] },
+      ]);
+    });
+
+    it.each([
+      ['opening tag', '<a href=https://example.com>', '', { type: 'root' }, 'a', { href: 'https://example.com' }, undefined],
+      ['paired tag', '<a href=https://example.com>Example</a>', '', { type: 'root' }, 'a', { href: 'https://example.com' }, 'Example'],
+      ['inline class', '<span class=unquoted-class>Example</span>', '', { type: 'root' }, 'span', { className: ['unquoted-class'] }, 'Example'],
+      ['block class', '<div class=unquoted-class>Example</div>', '', { type: 'root' }, 'div', { className: ['unquoted-class'] }, 'Example'],
+      ['self-closing tag', '<img src=https://example.com/image.png />', '', { type: 'root' }, 'img', { src: 'https://example.com/image.png' }, undefined],
+      ['attribute whitespace', '<div class = unquoted-class>Example</div>', '', { type: 'root' }, 'div', { className: ['unquoted-class'] }, 'Example'],
+      ['CRLF', '<div\r\n class=unquoted-class>\r\nExample\r\n</div>', '', { type: 'root' }, 'div', { className: ['unquoted-class'] }, undefined],
+      ['* emphasis', '*<a href=https://example.com>Example</a>*', 'em', { type: 'element', tagName: 'em' }, 'a', { href: 'https://example.com' }, 'Example'],
+      ['_ emphasis', '_<a href=https://example.com>Example</a>_', 'em', { type: 'element', tagName: 'em' }, 'a', { href: 'https://example.com' }, 'Example'],
+      ['nesting', 'Before <span class=outer><a href=https://example.com>Example</a></span> after', 'span', { type: 'element', tagName: 'span', properties: { className: ['outer'] } }, 'a', { href: 'https://example.com' }, 'Example'],
+      ['blank lines', '<div class=outer>\n\n<a href=https://example.com>Example</a>\n\n</div>', 'div', { type: 'element', tagName: 'div', properties: { className: ['outer'] } }, 'a', { href: 'https://example.com' }, 'Example'],
+      ['tab formatting', 'Before <a\thref\t=\thttps://example.com>Example</a> after', '', { type: 'root' }, 'a', { href: 'https://example.com' }, 'Example'],
+    ])('should parse the %s variant', (_case, source, parentTag, expectedParent, tagName, properties, value) => {
+      const tree = parse(source);
+      const parent = findElementByTagName(tree, parentTag) ?? tree;
+
+      expect(parent).toMatchObject(expectedParent);
+
+      expect(findElementByTagName(parent, tagName)).toMatchObject({
+        type: 'element',
+        tagName,
+        properties,
+        ...(value !== undefined ? { children: [{ type: 'text', value }] } : {}),
+      });
+    });
+
+    it('should leave an escaped opening tag as text', () => {
+      const tree = parse('Before \\<span class=inner>Example</span> after');
+
+      expect(findElementByTagName(tree, 'span')).toBeNull();
+      expect(extractText(tree)).toBe('Before <span class=inner>Example after');
+    });
+  });
 });

__tests__/lib/micromark/mdx-component.test.ts

@@ -0,0 +1,86 @@
+import type { Nodes, Root } from 'mdast';
+import type { Event } from 'micromark-util-types';
+
+import { fromMarkdown } from 'mdast-util-from-markdown';
+import { parse, postprocess, preprocess } from 'micromark';
+
+import { mdxComponentFromMarkdown } from '../../../lib/mdast-util/mdx-component';
+import { mdxComponent } from '../../../lib/micromark/mdx-component';
+
+const LINK_OPEN = '<a href=https://example.com>';
+const LINK = `${LINK_OPEN}Example</a>`;
+const FLOW = '<div data-url=https://example.com>\nExample\n</div>';
+
+const claimedComponents = (source: string): string[] =>
+  postprocess(parse({ extensions: [mdxComponent()] }).document().write(preprocess()(source, 'utf8', true)))
+    .filter(([event, token]: Event) => event === 'enter' && token.type === 'mdxComponent')
+    .map(([, token]: Event) => source.slice(token.start.offset, token.end.offset));
+
+const parseMdast = (source: string): Root =>
+  fromMarkdown(source, { extensions: [mdxComponent()], mdastExtensions: [mdxComponentFromMarkdown()] });
+
+const mdastTypes = (node: Nodes): string[] => [
+  node.type,
+  ...('children' in node ? node.children.flatMap(mdastTypes) : []),
+];
+
+describe('RM-16375 unquoted native HTML attributes', () => {
+  it.each([
+    'Pass <key=value> to the CLI.',
+    'Compare <a = b> and <c = d>.',
+  ])('does not claim tag-like prose: %s', source => {
+    expect(claimedComponents(source)).toStrictEqual([]);
+    expect(mdastTypes(parseMdast(source))).toStrictEqual(['root', 'paragraph', 'text']);
+  });
+
+  it.each([
+    '<Image src=https://example.com/a.png?w=1 align=center />',
+    '<Image value=foo"bar" />',
+    "<Image value=foo'bar' />",
+    '<Image value=foo`bar` />',
+  ])('continues to claim PascalCase components with permissive attribute values: %s', source => {
+    expect(claimedComponents(source)).toStrictEqual([source]);
+    expect(mdastTypes(parseMdast(source))).toStrictEqual(['root', 'html']);
+  });
+
+  it.each([
+    ['flow', FLOW, [FLOW], ['root', 'html']],
+    ['text', `Before ${LINK} after`, [LINK_OPEN], ['root', 'paragraph', 'text', 'html', 'text', 'html', 'text']],
+  ])('claims %s input and emits the expected MDAST shape', (_context, source, claims, types) => {
+    expect(claimedComponents(source)).toStrictEqual(claims);
+    expect(mdastTypes(parseMdast(source))).toStrictEqual(types);
+  });
+
+  it.each([
+    '<img src=https://example.com/image.png />',
+    '<div class = unquoted-class>Example</div>',
+    '<div\r\n class=unquoted-class>\r\nExample\r\n</div>',
+  ])('claims valid whitespace, line-ending, and self-closing variants: %s', source => {
+    expect(claimedComponents(source)).toStrictEqual([source]);
+  });
+
+  it.each([
+    '<a href=>',
+    '<a href==value>',
+    ...['"', "'", '<', '=', '`'].map(delimiter => `<a href=value${delimiter}suffix>`),
+  ])('does not claim malformed or forbidden unquoted values: %s', source => {
+    expect(claimedComponents(source)).toStrictEqual([]);
+  });
+
+  it.each([
+    ['flow emphasis', '<div class=outer>*Example*</div>', ['<div class=outer>*Example*</div>'], ['root', 'html']],
+    ['text * emphasis', `*${LINK}*`, [LINK_OPEN], ['root', 'paragraph', 'emphasis', 'html', 'text', 'html']],
+    ['text _ emphasis', `_${LINK}_`, [LINK_OPEN], ['root', 'paragraph', 'emphasis', 'html', 'text', 'html']],
+    ['escaped flow opening', '\\<div class=outer>Example</div>', [], ['root', 'paragraph', 'text', 'html']],
+    ['escaped text opening', 'Before \\<span class=inner>Example</span> after', [], ['root', 'paragraph', 'text', 'html', 'text']],
+    ['flow nesting', `<div class=outer>${LINK}</div>`, [`<div class=outer>${LINK}</div>`], ['root', 'html']],
+    ['text nesting', `Before <span class=outer>${LINK}</span> after`, ['<span class=outer>', LINK_OPEN], ['root', 'paragraph', 'text', 'html', 'html', 'text', 'html', 'html', 'text']],
+    ['flow blank lines', `<div class=outer>\n\n${LINK}\n\n</div>`, [`<div class=outer>\n\n${LINK}\n\n</div>`], ['root', 'html']],
+    ['text blank lines', `Before\n\n${LINK}\n\nAfter`, [LINK_OPEN], ['root', 'paragraph', 'text', 'paragraph', 'html', 'text', 'html', 'paragraph', 'text']],
+    ['flow formatting', '<div\n class = outer>\nExample\n</div>', ['<div\n class = outer>\nExample\n</div>'], ['root', 'html']],
+    ['text formatting', 'Before <a\thref\t=\thttps://example.com>Example</a> after', ['<a\thref\t=\thttps://example.com>'], ['root', 'paragraph', 'text', 'html', 'text', 'html', 'text']],
+  ])('handles %s with the expected claim and MDAST shape', (_case, source, claims, types) => {
+    expect(claimedComponents(source)).toStrictEqual(claims);
+    expect(mdastTypes(parseMdast(source))).toStrictEqual(types);
+  });
+});

lib/micromark/mdx-component/syntax.ts

@@ -5,7 +5,12 @@ import { markdownLineEnding, markdownSpace } from 'micromark-util-character';
 import { htmlBlockNames, htmlRawNames } from 'micromark-util-html-tag-name';
 import { codes, types } from 'micromark-util-symbol';
 
-import { FOREIGN_CONTENT_TAGS, HTML_TABLE_STRUCTURE_TAGS, HTML_VOID_ELEMENTS } from '../../../utils/common-html-words';
+import {
+  FOREIGN_CONTENT_TAGS,
+  HTML_TABLE_STRUCTURE_TAGS,
+  HTML_VOID_ELEMENTS,
+  STANDARD_HTML_TAGS,
+} from '../../../utils/common-html-words';
 import { INLINE_COMPONENT_TAGS, TOKENIZER_MDX_COMPONENT_EXCLUDED_TAGS } from '../../constants';
 
 import { markupOnlyContinuation, nonLazyContinuationStart } from './continuation-checks';
@@ -93,13 +98,14 @@ function createTokenize(mode: 'flow' | 'text') {
     let tagName = '';
     let depth = 0;
     let closingTagName = '';
-    // For lowercase tags we only want to claim the block if it uses JSX
-    // attribute expression syntax (`attr={...}`). Plain HTML should fall
-    // through to CommonMark html-flow. Flow mode claims any PascalCase block
-    // component; text mode claims only inline PascalCase components
-    // (INLINE_COMPONENT_TAGS — Anchor, Glossary), also brace-gated.
+    // Lowercase tags are claimed when they use JSX attribute expressions or
+    // unquoted HTML attribute values, preventing URL-like values from being
+    // split into text and autolink nodes elsewhere in the MDXish pipeline.
     let isLowercaseTag = false;
     let sawBraceAttr = false;
+    let sawUnquotedAttr = false;
+    let awaitingAttrValue = false;
+    let sawAttributeName = false;
 
     // A plain lowercase block tag claimed without a `{…}` attribute, gated by
     // `plainClaimLineStart`: after a blank line it may only continue on a tag line.
@@ -316,6 +322,7 @@ function createTokenize(mode: 'flow' | 'text') {
         tagName = String.fromCharCode(code);
         isLowercaseTag = false;
         sawBraceAttr = false;
+        sawUnquotedAttr = false;
         effects.consume(code);
         return tagNameRest;
       }
@@ -327,6 +334,7 @@ function createTokenize(mode: 'flow' | 'text') {
         tagName = String.fromCharCode(code);
         isLowercaseTag = true;
         sawBraceAttr = false;
+        sawUnquotedAttr = false;
         effects.consume(code);
         return tagNameRest;
       }
@@ -368,9 +376,12 @@ function createTokenize(mode: 'flow' | 'text') {
       if (code === null) return nok(code);
 
       // Everything except a flow-mode PascalCase block component must carry a
-      // `{…}` brace attribute to be claimed; plain HTML falls through to
-      // CommonMark.
-      const requiresBraceAttr = isLowercaseTag || !isFlow;
+      // `{…}` expression or unquoted attribute to be claimed; quoted HTML falls
+      // through to CommonMark.
+      const requiresSpecialAttr = isLowercaseTag || !isFlow;
+      const hasClaimableAttr =
+        sawBraceAttr ||
+        (sawUnquotedAttr && STANDARD_HTML_TAGS.has(tagName) && (!isFlow || htmlFlowTagNames.has(tagName)));
 
       if (markdownLineEnding(code)) {
         if (!isFlow) return nok(code);
@@ -380,38 +391,99 @@ function createTokenize(mode: 'flow' | 'text') {
 
       // Self-closing />
       if (code === codes.slash) {
-        if (requiresBraceAttr && !sawBraceAttr) return nok(code);
+        if (awaitingAttrValue) {
+          awaitingAttrValue = false;
+          sawUnquotedAttr = true;
+          effects.consume(code);
+          return inUnquotedAttr;
+        }
+        if (requiresSpecialAttr && !hasClaimableAttr) return nok(code);
         effects.consume(code);
         return selfCloseGt;
       }
 
       // End of opening tag
       if (code === codes.greaterThan) {
-        if (requiresBraceAttr && !sawBraceAttr && !claimBraceLessTag()) return nok(code);
+        if (awaitingAttrValue) return nok(code);
+        if (requiresSpecialAttr && !hasClaimableAttr && !claimBraceLessTag()) return nok(code);
+        if (isFlow && isLowercaseTag && sawUnquotedAttr && !sawBraceAttr && plainBlockClaimTagNames.has(tagName)) {
+          isPlainBlockClaim = true;
+        }
         effects.consume(code);
+        if (!isFlow && isLowercaseTag && sawUnquotedAttr && !sawBraceAttr) return doneOpeningTag;
         onOpenerLine = isFlow;
         return pendingBlockWrapperClaim ? blockWrapperOpenerRest : body;
       }
 
       // Quoted attribute value
       if (code === codes.quotationMark || code === codes.apostrophe) {
+        awaitingAttrValue = false;
+        sawAttributeName = false;
         quoteChar = code;
         effects.consume(code);
         return inQuotedAttr;
       }
 
       // JSX expression attribute
       if (code === codes.leftCurlyBrace) {
+        awaitingAttrValue = false;
+        sawAttributeName = false;
         braceDepth = 1;
         sawBraceAttr = true;
         effects.consume(code);
         return inBraceExpr;
       }
 
+      if (code === codes.equalsTo) {
+        if (!isLowercaseTag) {
+          effects.consume(code);
+          return afterOpenTagName;
+        }
+        if (awaitingAttrValue || !sawAttributeName) return nok(code);
+        awaitingAttrValue = true;
+        sawAttributeName = false;
+        effects.consume(code);
+        return afterOpenTagName;
+      }
+
+      if (awaitingAttrValue && !markdownSpace(code)) {
+        if (code === codes.lessThan || code === codes.graveAccent) return nok(code);
+        awaitingAttrValue = false;
+        sawUnquotedAttr = true;
+        effects.consume(code);
+        return inUnquotedAttr;
+      }
+
+      if (isLowercaseTag && !markdownSpace(code)) sawAttributeName = true;
       effects.consume(code);
       return afterOpenTagName;
     }
 
+    function inUnquotedAttr(code: Code): State | undefined {
+      if (
+        code === null ||
+        code === codes.quotationMark ||
+        code === codes.apostrophe ||
+        code === codes.lessThan ||
+        code === codes.equalsTo ||
+        code === codes.graveAccent
+      ) {
+        return nok(code);
+      }
+      if (code === codes.greaterThan || markdownLineEnding(code) || markdownSpace(code)) {
+        sawAttributeName = false;
+        return afterOpenTagName(code);
+      }
+      effects.consume(code);
+      return inUnquotedAttr;
+    }
+
+    function doneOpeningTag(code: Code): State | undefined {
+      effects.exit('mdxComponentData');
+      effects.exit('mdxComponent');
+      return ok(code);
+    }
+
     function inQuotedAttr(code: Code): State | undefined {
       if (code === null) return nok(code);