fix: hide hidden pages from llms.txt (#38533)

Pages that are disabled in navigation/site rendering are still shown in
llms.txt. This change filters those pages out of the llms.txt build as
well.

Co-authored-by: Chris Chinchilla <chris.ward@supabase.io>

apps/docs/scripts/llms.ts

@@ -3,6 +3,8 @@ import './utils/dotenv.js'
 import 'dotenv/config'
 import fs from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
+
+import { isFeatureEnabled } from 'common/enabled-features'
 import {
   fetchCliLibReferenceSource,
   fetchCSharpLibReferenceSource,
@@ -22,8 +24,17 @@ interface Source {
    */
   relPath: string
   fetch: () => Promise<SearchSource[]>
+  enabled: boolean
 }
 
+const {
+  sdkCsharp: sdkCsharpEnabled,
+  sdkDart: sdkDartEnabled,
+  sdkKotlin: sdkKotlinEnabled,
+  sdkPython: sdkPythonEnabled,
+  sdkSwift: sdkSwiftEnabled,
+} = isFeatureEnabled(['sdk:csharp', 'sdk:dart', 'sdk:kotlin', 'sdk:python', 'sdk:swift'])
+
 function toLink(source: Source) {
   return `[${source.title}](https://supabase.com/${source.relPath})`
 }
@@ -33,6 +44,7 @@ const SOURCES: Source[] = [
     title: 'Supabase Guides',
     relPath: 'llms/guides.txt',
     fetch: fetchGuideSources,
+    enabled: true,
   },
   {
     title: 'Supabase Reference (JavaScript)',
@@ -41,6 +53,7 @@ const SOURCES: Source[] = [
       (await fetchJsLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: true,
   },
   {
     title: 'Supabase Reference (Dart)',
@@ -49,6 +62,7 @@ const SOURCES: Source[] = [
       (await fetchDartLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: sdkDartEnabled,
   },
   {
     title: 'Supabase Reference (Swift)',
@@ -57,6 +71,7 @@ const SOURCES: Source[] = [
       (await fetchSwiftLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: sdkSwiftEnabled,
   },
   {
     title: 'Supabase Reference (Kotlin)',
@@ -65,6 +80,7 @@ const SOURCES: Source[] = [
       (await fetchKtLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: sdkKotlinEnabled,
   },
   {
     title: 'Supabase Reference (Python)',
@@ -73,6 +89,7 @@ const SOURCES: Source[] = [
       (await fetchPythonLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: sdkPythonEnabled,
   },
   {
     title: 'Supabase Reference (C#)',
@@ -81,6 +98,7 @@ const SOURCES: Source[] = [
       (await fetchCSharpLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: sdkCsharpEnabled,
   },
   {
     title: 'Supabase CLI Reference',
@@ -89,11 +107,14 @@ const SOURCES: Source[] = [
       (await fetchCliLibReferenceSource()).filter(
         (item): item is SearchSource => item !== undefined
       ),
+    enabled: true,
   },
 ]
 
 async function generateMainLlmsTxt() {
-  const sourceLinks = SOURCES.map((source) => `- ${toLink(source)}`).join('\n')
+  const sourceLinks = SOURCES.filter((source) => source.enabled !== false)
+    .map((source) => `- ${toLink(source)}`)
+    .join('\n')
   const fullText = `# Supabase Docs\n\n${sourceLinks}`
   fs.writeFile('public/llms.txt', fullText)
 }
@@ -114,7 +135,10 @@ async function generateSourceLlmsTxt(sourceDefn: Source) {
 async function generateLlmsTxt() {
   try {
     await fs.mkdir('public/llms', { recursive: true })
-    await Promise.all([generateMainLlmsTxt(), ...SOURCES.map(generateSourceLlmsTxt)])
+    await Promise.all([
+      generateMainLlmsTxt(),
+      ...SOURCES.filter((source) => source.enabled !== false).map(generateSourceLlmsTxt),
+    ])
   } catch (err) {
     console.error(err)
   }

apps/docs/scripts/search/sources/index.ts

@@ -1,5 +1,4 @@
-import { sep } from 'node:path'
-
+import { GuideModelLoader } from '../../../resources/guide/guideModelLoader.js'
 import {
   GitHubDiscussionLoader,
   type GitHubDiscussionSource,
@@ -16,9 +15,6 @@ import {
   OpenApiReferenceLoader,
   type OpenApiReferenceSource,
 } from './reference-doc.js'
-import { walk } from './util.js'
-
-const ignoredFiles = ['pages/404.mdx']
 
 export type SearchSource =
   | MarkdownSource
@@ -30,18 +26,9 @@ export type SearchSource =
   | LintWarningsGuideSource
 
 export async function fetchGuideSources() {
-  return (
-    await Promise.all(
-      (await walk('content/guides'))
-        .filter(
-          ({ path }) =>
-            /\.mdx?$/.test(path) &&
-            !ignoredFiles.includes(path) &&
-            !path.split(sep).some((part) => part.startsWith('_'))
-        )
-        .map((entry) => new MarkdownLoader('guide', entry.path, { yaml: true }).load())
-    )
-  ).flat()
+  const guides = (await GuideModelLoader.allFromFs()).unwrapLeft()
+
+  return guides.map((guide) => MarkdownLoader.fromGuideModel('guide', guide))
 }
 
 export async function fetchOpenApiReferenceSource() {

apps/docs/scripts/search/sources/markdown.ts

@@ -1,4 +1,4 @@
-import { SubsectionModel } from '../../../resources/guide/guideModel.js'
+import { GuideModel, SubsectionModel } from '../../../resources/guide/guideModel.js'
 import { GuideModelLoader } from '../../../resources/guide/guideModelLoader.js'
 import { BaseLoader, BaseSource } from './base.js'
 
@@ -32,6 +32,16 @@ export class MarkdownLoader extends BaseLoader {
       ),
     ]
   }
+
+  static fromGuideModel(source: string, guide: GuideModel): MarkdownSource {
+    const path = guide.href ? guide.href.replace('https://supabase.com/docs', '') : ''
+
+    return new MarkdownSource(source, path, guide.content ?? '', {
+      checksum: guide.checksum,
+      meta: guide.metadata,
+      sections: guide.subsections,
+    })
+  }
 }
 
 export class MarkdownSource extends BaseSource {