chore:new AST based script supporting current markdown for LLMs (#46949)

apps/docs/internals/generate-guides-markdown.ts

@@ -1,198 +1,160 @@
-import fs from 'node:fs'
+import fs from 'node:fs/promises'
 import path from 'node:path'
 import { globby } from 'globby'
 import matter from 'gray-matter'
+import type { Content, Parent, Root } from 'mdast'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx'
+import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx-jsx'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { gfm } from 'micromark-extension-gfm'
+import { mdxjs } from 'micromark-extension-mdxjs'
 
-import { getInternalLinkBaseUrl, prefixInternalLinks, withDocsBasePath } from './internal-links'
+import { getInternalLinkBaseUrl, prefixInternalLinks } from './internal-links'
+import { Link } from './markdown-schema/Link'
+import { Panel } from './markdown-schema/Panel'
+import { StepHike } from './markdown-schema/StepHike'
+import { TabPanel } from './markdown-schema/TabPanel'
+import { Admonition } from './markdown-schema/Admonition'
 
 const PARTIALS_DIR = path.join(process.cwd(), 'content', '_partials')
 
+type JsxNode = MdxJsxFlowElement | MdxJsxTextElement
+type Props = Record<string, unknown>
+
 /**
- * Reads <$Partial path="..." /> tags and replaces them with the file contents.
- * Recurses to handle nested partials.
+ * A handler converts a single MDX component into a markdown string. It receives
+ * the component's props, the already-serialized markdown of its children, and
+ * the raw AST node (escape hatch for handlers that need to inspect structure).
+ *
+ * Any component not in the schema is treated as `({ children }) => children`,
+ * i.e. the wrapper is dropped and its children are kept as-is.
  */
-async function inlinePartials(content: string): Promise<string> {
-  const partialRegex = /<\$Partial\s+path="([^"]+)"[^/]*\/>/g
-  const matches = [...content.matchAll(partialRegex)]
-  for (const [fullMatch, partialPath] of matches) {
-    try {
-      const raw = await fs.promises.readFile(path.join(PARTIALS_DIR, partialPath), 'utf8')
-      const { content: partialBody } = matter(raw)
-      const inlined = await inlinePartials(partialBody)
-      content = content.replace(fullMatch, inlined)
-    } catch {
-      content = content.replace(fullMatch, '')
-    }
+type ComponentHandler = (ctx: { props: Props; children: string; node: JsxNode }) => string
+type ComponentSchema = Record<string, ComponentHandler>
+
+const PARSE_OPTIONS = {
+  extensions: [mdxjs(), gfm()],
+  mdastExtensions: [mdxFromMarkdown(), gfmFromMarkdown()],
+}
+const SERIALIZE_OPTIONS = {
+  extensions: [mdxToMarkdown(), gfmToMarkdown()],
+  bullet: '-' as const,
+  listItemIndent: 'one' as const,
+}
+
+const parseMdx = (source: string): Root => fromMarkdown(source, PARSE_OPTIONS)
+const serializeMdx = (tree: Parent): string => toMarkdown(tree as Root, SERIALIZE_OPTIONS)
+
+const defaultHandler: ComponentHandler = ({ children }) => children
+
+const isJsx = (n: Content): n is JsxNode =>
+  n.type === 'mdxJsxFlowElement' || n.type === 'mdxJsxTextElement'
+
+function propsFrom(node: JsxNode): Props {
+  const props: Props = {}
+  for (const attr of node.attributes) {
+    if (attr.type !== 'mdxJsxAttribute') continue
+    if (attr.value == null) props[attr.name] = true
+    else if (typeof attr.value === 'string') props[attr.name] = attr.value
+    else props[attr.name] = attr.value.value
   }
-  return content
-}
-
-/** Remove the minimum common leading whitespace from all non-empty lines. */
-function dedentBlock(text: string): string {
-  const lines = text.split('\n')
-  const nonEmpty = lines.filter((l) => /\S/.test(l))
-  if (!nonEmpty.length) return text
-  const minIndent = Math.min(...nonEmpty.map((l) => (l.match(/^([ \t]*)/) ?? ['', ''])[1].length))
-  if (!minIndent) return text
-  return lines.map((l) => l.slice(minIndent)).join('\n')
+  return props
 }
 
 /**
- * Converts StepHikeCompact components to markdown ordered lists.
- * Each step becomes a numbered item: the title (from Details) is bolded on the item
- * line, and the full step body (Details + Code) is dedented and appended below.
- * Remaining JSX tags inside the body are later stripped by stripJsxTags.
+ * Replaces every `<$Partial path="..." />` in the tree with the parsed AST of
+ * the referenced file. Recurses so partials may include other partials.
  */
-function convertStepHike(content: string): string {
-  return content.replace(/<StepHikeCompact>([\s\S]*?)<\/StepHikeCompact>/g, (_, body) => {
-    const items: string[] = []
-    const stepRe = /<StepHikeCompact\.Step[^>]*>([\s\S]*?)<\/StepHikeCompact\.Step>/g
-    let stepNum = 1
-    let m: RegExpExecArray | null
-    while ((m = stepRe.exec(body)) !== null) {
-      const stepBody = m[1]
-      const titleMatch = stepBody.match(/<StepHikeCompact\.Details[^>]+title="([^"]*)"/)
-      const title = titleMatch ? titleMatch[1] : ''
-      // Dedent the entire step body so nested JSX indentation is removed.
-      // Remaining component tags (Details, Code, Admonition…) are stripped later.
-      const inner = dedentBlock(stepBody).trim()
-      const item = title ? `${stepNum}. **${title}**\n\n${inner}` : `${stepNum}. ${inner}`
-      items.push(item)
-      stepNum++
+async function inlinePartials(parent: Parent): Promise<void> {
+  const next: Content[] = []
+  for (const child of parent.children as Content[]) {
+    if (isJsx(child) && child.name === '$Partial') {
+      const partialPath = String(propsFrom(child).path ?? '')
+      try {
+        const raw = await fs.readFile(path.join(PARTIALS_DIR, partialPath), 'utf8')
+        const subtree = parseMdx(matter(raw).content)
+        await inlinePartials(subtree)
+        next.push(...(subtree.children as Content[]))
+      } catch {
+        // missing or broken partials are silently dropped
+      }
+      continue
     }
-    return items.join('\n\n')
-  })
-}
-
-/**
- * Extracts a `title` prop value from a JSX opening-tag's attribute text.
- * Supports `title="..."`, `title='...'`, and `title={<jsx>...</jsx>}` forms.
- * For JSX titles, strips inline tags and collapses whitespace so the text
- * content remains.
- */
-function extractTitleAttr(attrs: string): string | null {
-  const strMatch = attrs.match(/\btitle=(?:"([^"]+)"|'([^']+)')/)
-  if (strMatch) return strMatch[1] ?? strMatch[2]
-
-  const exprIdx = attrs.indexOf('title={')
-  if (exprIdx === -1) return null
-
-  let i = exprIdx + 'title={'.length
-  const inner: string[] = []
-  let depth = 1
-  while (i < attrs.length && depth > 0) {
-    const ch = attrs[i]
-    if (ch === '{') depth++
-    else if (ch === '}') {
-      depth--
-      if (depth === 0) break
-    }
-    inner.push(ch)
-    i++
+    if ('children' in child) await inlinePartials(child as Parent)
+    next.push(child)
   }
-  return inner
-    .join('')
-    .replace(/<[^>]+>/g, ' ')
-    .replace(/\s+/g, ' ')
-    .trim()
+  parent.children = next as Parent['children']
 }
 
 /**
- * Converts `<Link href="..."><GlassPanel|IconPanel title="...">desc</...></Link>`
- * blocks into markdown bullet lines: `- [title](href). description`. Without
- * this, the rendered output keeps prop syntax (className, passHref, etc.) since
- * stripping JSX tags discards inner text from props but leaves panel children
- * floating without context. Applied to all guides so resource-card grids render
- * consistently in the markdown view.
+ * Walks the tree bottom-up. For each JSX element, runs its schema handler (or
+ * the default) and replaces the node with an `html` node holding the result.
+ * The `html` type passes through `mdast-util-to-markdown` verbatim, so whatever
+ * markdown the handler returns lands in the final output unchanged.
  */
-function convertLinkPanels(content: string): string {
-  return content.replace(/<Link\b([\s\S]*?)>([\s\S]*?)<\/Link>/g, (full, linkAttrs, body) => {
-    const hrefMatch = linkAttrs.match(/\bhref="([^"]+)"/)
-    if (!hrefMatch) return full
-    const href = withDocsBasePath(hrefMatch[1])
-
-    const panelOpen = body.match(/<(GlassPanel|IconPanel)\b/)
-    if (!panelOpen) return full
-    const panelName = panelOpen[1]
-    const panelStart = panelOpen.index ?? 0
-
-    // Walk past the opening tag, respecting JSX expression braces so attribute
-    // values like `title={<span>...</span>}` don't terminate parsing early.
-    let i = panelStart + panelOpen[0].length
-    let depth = 0
-    while (i < body.length) {
-      const ch = body[i]
-      if (ch === '{') depth++
-      else if (ch === '}') depth--
-      else if (ch === '>' && depth === 0) break
-      i++
+function applySchema(parent: Parent, schema: ComponentSchema): void {
+  for (const child of parent.children as Content[]) {
+    if ('children' in child) applySchema(child as Parent, schema)
+  }
+  const next: Content[] = []
+  for (const child of parent.children as Content[]) {
+    if (
+      child.type === 'mdxFlowExpression' ||
+      child.type === 'mdxTextExpression' ||
+      child.type === 'mdxjsEsm'
+    ) {
+      continue
     }
-    if (i >= body.length) return full
-
-    const panelAttrs = body.slice(panelStart + panelOpen[0].length, i)
-    const closingTag = `</${panelName}>`
-    const closeIdx = body.indexOf(closingTag, i)
-    if (closeIdx === -1) return full
-
-    const title = extractTitleAttr(panelAttrs)
-    if (!title) return full
-
-    const description = body
-      .slice(i + 1, closeIdx)
-      .replace(/<[^>]+>/g, ' ')
-      .replace(/\s+/g, ' ')
-      .trim()
-
-    return `- [${title}](${href})${description ? `. ${description}` : ''}`
-  })
+    if (isJsx(child)) {
+      const handler = schema[child.name ?? ''] ?? defaultHandler
+      const children = serializeMdx({
+        type: 'root',
+        children: child.children as Root['children'],
+      }).trim()
+      const value = handler({ props: propsFrom(child), children, node: child })
+      next.push({ type: 'html', value } as Content)
+      continue
+    }
+    next.push(child)
+  }
+  parent.children = next as Parent['children']
 }
 
 /**
- * Strips JSX component tags (capitalized names, dot-notation, or $-prefixed)
- * while keeping their inner content. Also strips wrapper div and a elements.
- * Removes MDX JSX comment blocks. Strips unnecessary leading indentation from
- * non-code-block lines.
+ * Per-component overrides. Each handler receives `{ props, children, node }`
+ * and returns the markdown string that should replace the JSX element. Any
+ * component not listed is unwrapped (children are kept, wrapper is dropped).
  */
-function stripJsxTags(content: string): string {
-  // Remove MDX/JSX comments {/* ... */}
-  content = content.replace(/\{\/\*[\s\S]*?\*\/\}/g, '')
+const SCHEMA: ComponentSchema = {
+  Admonition,
+  Link,
+  GlassPanel: Panel,
+  IconPanel: Panel,
+  ...StepHike,
+  TabPanel,
+}
 
-  // Remove self-closing JSX components: <Component ... /> or <$Directive ... />
-  content = content.replace(/<[\$A-Z][\w.]*(?:\s[^>]*)?\s*\/>/gs, '')
+async function generateOne(filePath: string, linkBaseUrl: string): Promise<string> {
+  const raw = await fs.readFile(filePath, 'utf8')
+  const { content, data } = matter(raw)
 
-  // Remove opening JSX component tags (possibly multi-line): <Component ...>
-  content = content.replace(/<[\$A-Z][\w.]*(?:\s[^>]*)?\s*>/gs, '')
+  const tree = parseMdx(content)
+  await inlinePartials(tree)
+  applySchema(tree, SCHEMA)
+  const body = prefixInternalLinks(serializeMdx(tree), linkBaseUrl)
 
-  // Remove closing JSX component tags: </Component>
-  content = content.replace(/<\/[\$A-Z][\w.]*>/g, '')
+  const headerParts: string[] = []
+  if (data.title) headerParts.push(`# ${data.title}`)
+  if (data.subtitle) headerParts.push(String(data.subtitle))
+  // Add description only when differs from subtitle.
+  if (data.description && String(data.description) !== String(data.subtitle))
+    headerParts.push(String(data.description))
 
-  // Remove wrapper div and a elements used structurally in MDX (carry JSX props
-  // like className which are not valid HTML; inner content such as img is preserved)
-  content = content.replace(/<div(?:\s[^>]*)?\s*>/g, '')
-  content = content.replace(/<\/div>/g, '')
-  content = content.replace(/<a(?:\s[^>]*)?\s*>/g, '')
-  content = content.replace(/<\/a>/g, '')
+  const header = headerParts.join('\n\n')
 
-  // Split on fenced code blocks to handle prose and code separately.
-  // For prose (even segments): strip leading whitespace (removes JSX nesting indent).
-  // For code blocks (odd segments): dedent the body to remove JSX nesting indent while
-  // preserving relative code structure, then normalize the closing fence.
-  const segments = content.split(/(```[\s\S]*?```)/g)
-  content = segments
-    .map((seg, i) => {
-      if (i % 2 === 0) return seg.replace(/^[ \t]+/gm, '')
-      return seg.replace(
-        /^(```[^\n]*\n)([\s\S]*?)(\n[ \t]*```)$/,
-        (_, open, body) => open + dedentBlock(body) + '\n```'
-      )
-    })
-    .join('')
-
-  // Collapse lines that are only whitespace to empty lines, then deduplicate blank lines
-  content = content.replace(/^[^\S\n]+$/gm, '')
-  content = content.replace(/\n{3,}/g, '\n\n').trim()
-
-  return content
+  return header ? `${header}\n\n${body}` : body
 }
 
 async function generate() {
@@ -202,47 +164,30 @@ async function generate() {
 
   await Promise.all(
     files.map(async (filePath) => {
+      // content/guides/ai/vector-columns.mdx → public/markdown/guides/ai/vector-columns.md
+      // Placing under public/markdown/ ensures the file is served at /docs/guides/...
+      // matching the URL of the rendered page.
       const outPath = filePath
         .replace(/^content\/guides\//, 'public/markdown/guides/')
         .replace(/\.mdx$/, '.md')
 
       let output: string
       try {
-        const raw = await fs.promises.readFile(filePath, 'utf8')
-        const { content: rawContent, data } = matter(raw)
-
-        const withPartials = await inlinePartials(rawContent)
-        const withSteps = convertStepHike(withPartials)
-        const withLinks = convertLinkPanels(withSteps)
-        const stripped = stripJsxTags(withLinks)
-        const processed = prefixInternalLinks(stripped, linkBaseUrl)
-
-        const header = [
-          data.title ? `# ${data.title}` : '',
-          data.subtitle || data.description ? `\n${data.subtitle ?? data.description}` : '',
-        ]
-          .filter(Boolean)
-          .join('\n')
-
-        output = header ? `${header}\n\n${processed}` : processed
+        output = await generateOne(filePath, linkBaseUrl)
       } catch (err) {
         warnings++
         console.warn(
           `[warn] Failed to process ${filePath}: ${err instanceof Error ? err.message : err}`
         )
-        // Fall back to raw file content so the route still serves something
         try {
-          output = await fs.promises.readFile(filePath, 'utf8')
+          output = await fs.readFile(filePath, 'utf8')
         } catch {
           output = `<!-- failed to generate: ${filePath} -->`
         }
       }
 
-      // content/guides/ai/vector-columns.mdx → public/markdown/guides/ai/vector-columns.md
-      // Placing under public/markdown/ ensures the file is served at /docs/guides/...
-      // matching the exact URL of the rendered page.
-      await fs.promises.mkdir(path.dirname(outPath), { recursive: true })
-      await fs.promises.writeFile(outPath, output)
+      await fs.mkdir(path.dirname(outPath), { recursive: true })
+      await fs.writeFile(outPath, output)
     })
   )
 

apps/docs/internals/markdown-schema/Admonition.ts

@@ -0,0 +1,13 @@
+export const Admonition = ({
+  props,
+  children,
+}: {
+  props: Record<string, unknown>
+  children: string
+}): string => {
+  const type = props.type ? String(props.type) : 'note'
+  return `${capitalize(type)}: ${children}`
+}
+
+const capitalize = <T extends string>(s: T) =>
+  (s[0].toUpperCase() + s.slice(1)) as Capitalize<typeof s>

apps/docs/internals/markdown-schema/Link.ts

@@ -0,0 +1,12 @@
+import { withDocsBasePath } from '../internal-links'
+
+export const Link = ({
+  props,
+  children,
+}: {
+  props: Record<string, unknown>
+  children: string
+}): string => {
+  const href = withDocsBasePath(String(props.href ?? ''))
+  return `[${children}](${href})`
+}

apps/docs/internals/markdown-schema/Panel.ts

@@ -0,0 +1,3 @@
+export const Panel = ({ props }: { props: Record<string, unknown> }): string => {
+  return String(props.title)
+}

apps/docs/internals/markdown-schema/StepHike.ts

@@ -0,0 +1,16 @@
+type Props = Record<string, unknown>
+type Ctx = { props: Props; children: string }
+
+const Step = ({ props, children }: Ctx): string => {
+  const step = String(props.step ?? '').trim()
+  return step ? `${step}. ${children}` : children
+}
+
+const Details = ({ props, children }: Ctx): string => {
+  return props.title ? `**${String(props.title).trim()}**\n\n${children}` : children
+}
+
+export const StepHike = {
+  'StepHikeCompact.Step': Step,
+  'StepHikeCompact.Details': Details,
+}

apps/docs/internals/markdown-schema/TabPanel.ts

@@ -0,0 +1,9 @@
+export const TabPanel = ({
+  props,
+  children,
+}: {
+  props: Record<string, unknown>
+  children: string
+}): string => {
+  return props.label ? `**${String(props.label)}**\n\n${children}` : children
+}