[DOCS-951] [DOCS-952] Update llms and llms-full text files to match industry standards (#46166)

apps/docs/internals/generate-guides-markdown.ts

@@ -2,7 +2,6 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { globby } from 'globby'
 import matter from 'gray-matter'
-import { create as createTar } from 'tar'
 
 const PARTIALS_DIR = path.join(process.cwd(), 'content', '_partials')
 
@@ -63,6 +62,44 @@ function convertStepHike(content: string): string {
   })
 }
 
+/**
+ * For getting-started.mdx: replaces `{[ ...objects ].map(...)}` resource-card
+ * grids with a markdown bullet list of `[title](href), description`. Without
+ * this, the rendered output leaves raw JS code in the markdown since stripping
+ * JSX components doesn't touch JS expressions wrapped in `{...}`.
+ */
+function convertResourceLists(content: string): string {
+  return content.replace(/\{\s*\[\s*\{[\s\S]*?\},?\s*\][\s\S]*?\}\)\}/g, (block) => {
+    const arrMatch = block.match(/\[([\s\S]+?)\]\s*\.(?:filter|map)\b/)
+    if (!arrMatch) return block
+
+    // Collect top-level { ... } object literals from the array body.
+    const arr = arrMatch[1]
+    const objs: string[] = []
+    let depth = 0
+    let start = -1
+    for (let i = 0; i < arr.length; i++) {
+      if (arr[i] === '{') {
+        if (depth === 0) start = i
+        depth++
+      } else if (arr[i] === '}' && --depth === 0 && start !== -1) {
+        objs.push(arr.slice(start, i + 1))
+        start = -1
+      }
+    }
+
+    return objs
+      .map((o) => {
+        const title = o.match(/title:\s*['"`]([^'"`]+)['"`]/)?.[1]
+        const href = o.match(/href:\s*['"`]([^'"`]+)['"`]/)?.[1]
+        const desc = o.match(/description:\s*[`'"]([^`'"]+)[`'"]/)?.[1]
+        return title && href ? `- [${title}](${href})${desc ? `. ${desc}` : ''}` : ''
+      })
+      .filter(Boolean)
+      .join('\n')
+  })
+}
+
 /**
  * Strips JSX component tags (capitalized names, dot-notation, or $-prefixed)
  * while keeping their inner content. Also strips wrapper div and a elements.
@@ -128,7 +165,11 @@ async function generate() {
 
         const withPartials = await inlinePartials(rawContent)
         const withSteps = convertStepHike(withPartials)
-        const processed = stripJsxTags(withSteps)
+        const withLists =
+          filePath === 'content/guides/getting-started.mdx'
+            ? convertResourceLists(withSteps)
+            : withSteps
+        const processed = stripJsxTags(withLists)
 
         const header = [
           data.title ? `# ${data.title}` : '',
@@ -161,16 +202,6 @@ async function generate() {
 
   const summary = warnings ? ` (${warnings} with warnings)` : ''
   console.log(`Generated ${files.length} markdown files under public/docs/guides/${summary}`)
-
-  // Create a tar.gz archive of the generated docs, served at /docs/docs.tar.gz.
-  // Sorted entries, portable headers, and a fixed mtime keep the output deterministic.
-  const archivePath = 'public/docs.tar.gz'
-  const entries = (await globby(['**'], { cwd: 'public/docs' })).sort()
-  await createTar(
-    { gzip: true, file: archivePath, cwd: 'public/docs', portable: true, mtime: new Date() },
-    entries
-  )
-  console.log(`Created archive at ${archivePath}`)
 }
 
 generate()

apps/docs/internals/generate-gz-archive.ts

@@ -0,0 +1,16 @@
+import { globby } from 'globby'
+import { create as createTar } from 'tar'
+
+async function generate() {
+  // Create a tar.gz archive of the generated docs, served at /docs/docs.tar.gz.
+  // Sorted entries, portable headers, and a fixed mtime keep the output deterministic.
+  const archivePath = 'public/docs.tar.gz'
+  const entries = (await globby(['**'], { cwd: 'public/docs' })).sort()
+  await createTar(
+    { gzip: true, file: archivePath, cwd: 'public/docs', portable: true, mtime: new Date() },
+    entries
+  )
+  console.log(`Created archive at ${archivePath}`)
+}
+
+generate()

apps/docs/package.json

@@ -8,6 +8,7 @@
     "build:analyze": "ANALYZE=true next build",
     "build:llms": "tsx --conditions=react-server ./scripts/llms.ts",
     "build:guides-markdown": "tsx ./internals/generate-guides-markdown.ts",
+    "build:gz-archive": "tsx ./internals/generate-gz-archive.ts",
     "build:sitemap": "tsx ./internals/generate-sitemap.ts",
     "clean": "rimraf .next .turbo node_modules features/docs/generated examples __generated__",
     "codegen:examples": "shx cp -r ../../examples ./examples",
@@ -26,7 +27,7 @@
     "lint": "eslint .",
     "lint:mdx": "supa-mdx-lint content --config ../../supa-mdx-lint.config.toml",
     "postbuild": "pnpm run build:sitemap && pnpm run build:llms && ./../../scripts/upload-static-assets.sh",
-    "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:guides-markdown",
+    "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:guides-markdown && pnpm run build:gz-archive",
     "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples",
     "preembeddings": "pnpm run codegen:references",
     "preinstall": "npx only-allow pnpm",

apps/www/app/llms-full.txt/route.ts

@@ -1,3 +1,5 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
 import { isFeatureEnabled } from 'common/enabled-features'
 
 import { MD_CONTENT } from '@/app/api-v2/md/content.generated'
@@ -11,6 +13,13 @@ interface Source {
   enabled: boolean
 }
 
+/**
+ * Resolved relative to apps/www (process.cwd() at runtime). The directory is
+ * included in the serverless bundle via outputFileTracingIncludes in
+ * next.config.mjs so reads work on Vercel.
+ */
+const GUIDES_MD_DIR = path.join(process.cwd(), '..', 'docs', 'public', 'docs', 'guides')
+
 function getSources(): Source[] {
   const { sdkCsharp, sdkDart, sdkKotlin, sdkPython, sdkSwift } = isFeatureEnabled([
     'sdk:csharp',
@@ -21,7 +30,6 @@ function getSources(): Source[] {
   ])
 
   return [
-    { title: 'Supabase Guides', slug: 'guides', enabled: true },
     { title: 'Supabase Reference (JavaScript)', slug: 'js', enabled: true },
     { title: 'Supabase Reference (Dart)', slug: 'dart', enabled: sdkDart },
     { title: 'Supabase Reference (Swift)', slug: 'swift', enabled: sdkSwift },
@@ -32,6 +40,17 @@ function getSources(): Source[] {
   ]
 }
 
+async function readAllGuideMarkdown(): Promise<string> {
+  const entries = await fs.readdir(GUIDES_MD_DIR, { recursive: true, withFileTypes: true })
+  const mdFilePaths = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+    .map((entry) => path.join(entry.parentPath, entry.name))
+    .sort()
+
+  const contents = await Promise.all(mdFilePaths.map((filePath) => fs.readFile(filePath, 'utf-8')))
+  return contents.join('\n\n---\n\n')
+}
+
 // Order is set by scripts/generateMdContent.mjs (homepage first, rest
 // alphabetical). pricing is appended here since it's dynamic.
 async function readProductOverviews(): Promise<string> {
@@ -55,19 +74,24 @@ export async function GET() {
   const sources = getSources()
   const enabledSources = sources.filter((source) => source.enabled)
 
-  const [productContent, ...sourceContents] = await Promise.all([
+  const [productContent, guidesContent, ...sourceContents] = await Promise.all([
     readProductOverviews(),
+    readAllGuideMarkdown(),
     ...enabledSources.map(async (source) => {
       const text = await fetchSourceContent(source.slug)
       return { title: source.title, text }
     }),
   ])
 
-  const docsSection = sourceContents
+  const referenceSection = sourceContents
     .filter((s): s is { title: string; text: string } => s.text !== null)
     .map(({ title, text }) => `# ${title}\n\n${text}`)
     .join('\n\n---\n\n')
 
+  const docsSection = [`# Supabase Guides\n\n${guidesContent}`, referenceSection].join(
+    '\n\n---\n\n'
+  )
+
   const content = [
     '# Supabase',
     '',

apps/www/app/llms.txt/route.ts

@@ -1,4 +1,7 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
 import { isFeatureEnabled } from 'common/enabled-features'
+import matter from 'gray-matter'
 
 export const dynamic = 'force-dynamic'
 
@@ -8,7 +11,44 @@ interface Source {
   enabled: boolean
 }
 
-function getSources(): Source[] {
+/**
+ * Resolved relative to apps/www (process.cwd() at runtime). The directory is
+ * included in the serverless bundle via outputFileTracingIncludes in
+ * next.config.mjs so this readdir works on Vercel.
+ */
+const GUIDES_CONTENT_DIR = path.join(process.cwd(), '..', 'docs', 'content', 'guides')
+
+async function readFrontmatterTitle(dirName: string): Promise<string | null> {
+  try {
+    const mdxPath = path.join(GUIDES_CONTENT_DIR, `${dirName}.mdx`)
+    const raw = await fs.readFile(mdxPath, 'utf-8')
+    const { data } = matter(raw)
+    return typeof data.title === 'string' && data.title.length > 0 ? data.title : null
+  } catch {
+    return null
+  }
+}
+
+async function getGuideSources(): Promise<Source[]> {
+  const entries = await fs.readdir(GUIDES_CONTENT_DIR, { withFileTypes: true })
+  const dirNames = entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort()
+
+  return Promise.all(
+    dirNames.map(async (dirName) => {
+      const frontmatterTitle = await readFrontmatterTitle(dirName)
+      return {
+        title: `Supabase - ${frontmatterTitle ?? dirName}`,
+        relPath: `docs/guides/${dirName}.md`,
+        enabled: true,
+      }
+    })
+  )
+}
+
+async function getSources(): Promise<Source[]> {
   const { sdkCsharp, sdkDart, sdkKotlin, sdkPython, sdkSwift } = isFeatureEnabled([
     'sdk:csharp',
     'sdk:dart',
@@ -17,8 +57,10 @@ function getSources(): Source[] {
     'sdk:swift',
   ])
 
+  const guideSources = await getGuideSources()
+
   return [
-    { title: 'Supabase Guides', relPath: 'llms/guides.txt', enabled: true },
+    ...guideSources,
     { title: 'Supabase Reference (JavaScript)', relPath: 'llms/js.txt', enabled: true },
     { title: 'Supabase Reference (Dart)', relPath: 'llms/dart.txt', enabled: sdkDart },
     { title: 'Supabase Reference (Swift)', relPath: 'llms/swift.txt', enabled: sdkSwift },
@@ -47,7 +89,7 @@ const PRODUCT_OVERVIEW_LINKS = [
 ].join('\n')
 
 export async function GET() {
-  const sources = getSources()
+  const sources = await getSources()
 
   const sourceLinks = sources
     .filter((source) => source.enabled)

apps/www/next.config.mjs

@@ -71,6 +71,17 @@ const nextConfig = {
       'public/**/*',
     ],
   },
+  /**
+   * /llms.txt enumerates top-level guide directories at runtime via fs.readdir
+   * on apps/docs/content/guides and reads each <dir>.mdx for its frontmatter
+   * title. /llms-full.txt recursively reads the generated .md files under
+   * apps/docs/public/docs/guides. Both directories live in a sibling app, so
+   * they have to be explicitly traced into this lambda's bundle.
+   */
+  outputFileTracingIncludes: {
+    '/llms.txt': ['../docs/content/guides/*'],
+    '/llms-full.txt': ['../docs/public/docs/guides/**/*.md'],
+  },
   reactStrictMode: true,
   images: {
     dangerouslyAllowSVG: false,

apps/www/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "dev": "pnpm run content:build && next --port 3000",
+    "prebuild": "pnpm --filter=docs run build:guides-markdown",
     "build": "pnpm run content:build && next build",
     "export": "next export",
     "start": "next start",

packages/ui-patterns/src/GlassPanel/index.tsx

@@ -19,6 +19,45 @@ interface Props {
   showIconBg?: boolean
 }
 
+const IconBackground = ({
+  children,
+  showIconBg,
+}: {
+  children: React.ReactNode
+  showIconBg?: boolean
+}) => (
+  <div
+    className={cn(
+      'shrink-0',
+      showIconBg ? 'bg-surface-75 border w-8 h-8 flex items-center justify-center rounded-sm' : ''
+    )}
+  >
+    {children}
+  </div>
+)
+
+const LogoComponent = ({
+  logoImage,
+  className,
+  title,
+}: {
+  title: string
+  logoImage: string
+  className?: string
+}) => (
+  <div className="relative box-content p-8 pb-0">
+    <div className="relative h-[33px] w-auto max-w-[145px]">
+      <Image
+        src={logoImage}
+        alt={title}
+        fill
+        sizes="100%"
+        className={cn('object-contain object-left', className)}
+      />
+    </div>
+  </div>
+)
+
 export const GlassPanel = ({
   title,
   icon,
@@ -36,31 +75,6 @@ export const GlassPanel = ({
   const showLogoInverse = logoInverse && resolvedTheme?.includes('dark')
   const showLogo = !showLogoInverse && logo
 
-  const IconBackground: React.FC<React.PropsWithChildren> = (props) => (
-    <div
-      className={cn(
-        'shrink-0',
-        showIconBg ? 'bg-surface-75 border w-8 h-8 flex items-center justify-center rounded-sm' : ''
-      )}
-    >
-      {props.children}
-    </div>
-  )
-
-  const LogoComponent = ({ logoImage, className }: { logoImage: string; className?: string }) => (
-    <div className="relative box-content p-8 pb-0">
-      <div className="relative h-[33px] w-auto max-w-[145px]">
-        <Image
-          src={logoImage}
-          alt={title}
-          fill
-          sizes="100%"
-          className={cn('object-contain object-left', className)}
-        />
-      </div>
-    </div>
-  )
-
   return (
     <div
       className={cn(
@@ -78,8 +92,10 @@ export const GlassPanel = ({
         className
       )}
     >
-      {showLogoInverse && <LogoComponent logoImage={logoInverse} className="opacity-50" />}
-      {showLogo && <LogoComponent logoImage={logo} className="opacity-75" />}
+      {showLogoInverse && (
+        <LogoComponent title={title} logoImage={logoInverse} className="opacity-50" />
+      )}
+      {showLogo && <LogoComponent title={title} logoImage={logo} className="opacity-75" />}
 
       {header && (
         <img
@@ -103,7 +119,7 @@ export const GlassPanel = ({
       >
         <div className="flex items-center gap-3">
           {icon && typeof icon === 'string' ? (
-            <IconBackground>
+            <IconBackground showIconBg={showIconBg}>
               <img
                 className="w-5"
                 alt={title}
@@ -113,7 +129,7 @@ export const GlassPanel = ({
               />
             </IconBackground>
           ) : (
-            icon && <IconBackground>{icon}</IconBackground>
+            icon && <IconBackground showIconBg={showIconBg}>{icon}</IconBackground>
           )}
           <p className="text-base text-foreground">{title}</p>
         </div>