From ec66a5d0962014bd68149ae9b65c038552b57073 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:47:27 -0400 Subject: [PATCH] fix: hide hidden pages from llms.txt (#38533) Pages that are disabled in navigation/site rendering are still shown in llms.txt. This change filters those pages out of the llms.txt build as well. Co-authored-by: Chris Chinchilla --- apps/docs/scripts/llms.ts | 28 ++++++++++++++++++-- apps/docs/scripts/search/sources/index.ts | 21 +++------------ apps/docs/scripts/search/sources/markdown.ts | 12 ++++++++- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/apps/docs/scripts/llms.ts b/apps/docs/scripts/llms.ts index 3452a72f140..9d68e431ed6 100644 --- a/apps/docs/scripts/llms.ts +++ b/apps/docs/scripts/llms.ts @@ -3,6 +3,8 @@ import './utils/dotenv.js' import 'dotenv/config' import fs from 'node:fs/promises' import { fileURLToPath } from 'node:url' + +import { isFeatureEnabled } from 'common/enabled-features' import { fetchCliLibReferenceSource, fetchCSharpLibReferenceSource, @@ -22,8 +24,17 @@ interface Source { */ relPath: string fetch: () => Promise + enabled: boolean } +const { + sdkCsharp: sdkCsharpEnabled, + sdkDart: sdkDartEnabled, + sdkKotlin: sdkKotlinEnabled, + sdkPython: sdkPythonEnabled, + sdkSwift: sdkSwiftEnabled, +} = isFeatureEnabled(['sdk:csharp', 'sdk:dart', 'sdk:kotlin', 'sdk:python', 'sdk:swift']) + function toLink(source: Source) { return `[${source.title}](https://supabase.com/${source.relPath})` } @@ -33,6 +44,7 @@ const SOURCES: Source[] = [ title: 'Supabase Guides', relPath: 'llms/guides.txt', fetch: fetchGuideSources, + enabled: true, }, { title: 'Supabase Reference (JavaScript)', @@ -41,6 +53,7 @@ const SOURCES: Source[] = [ (await fetchJsLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: true, }, { title: 'Supabase Reference (Dart)', @@ -49,6 +62,7 @@ const SOURCES: Source[] = [ (await fetchDartLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: sdkDartEnabled, }, { title: 'Supabase Reference (Swift)', @@ -57,6 +71,7 @@ const SOURCES: Source[] = [ (await fetchSwiftLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: sdkSwiftEnabled, }, { title: 'Supabase Reference (Kotlin)', @@ -65,6 +80,7 @@ const SOURCES: Source[] = [ (await fetchKtLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: sdkKotlinEnabled, }, { title: 'Supabase Reference (Python)', @@ -73,6 +89,7 @@ const SOURCES: Source[] = [ (await fetchPythonLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: sdkPythonEnabled, }, { title: 'Supabase Reference (C#)', @@ -81,6 +98,7 @@ const SOURCES: Source[] = [ (await fetchCSharpLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: sdkCsharpEnabled, }, { title: 'Supabase CLI Reference', @@ -89,11 +107,14 @@ const SOURCES: Source[] = [ (await fetchCliLibReferenceSource()).filter( (item): item is SearchSource => item !== undefined ), + enabled: true, }, ] async function generateMainLlmsTxt() { - const sourceLinks = SOURCES.map((source) => `- ${toLink(source)}`).join('\n') + const sourceLinks = SOURCES.filter((source) => source.enabled !== false) + .map((source) => `- ${toLink(source)}`) + .join('\n') const fullText = `# Supabase Docs\n\n${sourceLinks}` fs.writeFile('public/llms.txt', fullText) } @@ -114,7 +135,10 @@ async function generateSourceLlmsTxt(sourceDefn: Source) { async function generateLlmsTxt() { try { await fs.mkdir('public/llms', { recursive: true }) - await Promise.all([generateMainLlmsTxt(), ...SOURCES.map(generateSourceLlmsTxt)]) + await Promise.all([ + generateMainLlmsTxt(), + ...SOURCES.filter((source) => source.enabled !== false).map(generateSourceLlmsTxt), + ]) } catch (err) { console.error(err) } diff --git a/apps/docs/scripts/search/sources/index.ts b/apps/docs/scripts/search/sources/index.ts index c46c46a7986..9dc5541b274 100644 --- a/apps/docs/scripts/search/sources/index.ts +++ b/apps/docs/scripts/search/sources/index.ts @@ -1,5 +1,4 @@ -import { sep } from 'node:path' - +import { GuideModelLoader } from '../../../resources/guide/guideModelLoader.js' import { GitHubDiscussionLoader, type GitHubDiscussionSource, @@ -16,9 +15,6 @@ import { OpenApiReferenceLoader, type OpenApiReferenceSource, } from './reference-doc.js' -import { walk } from './util.js' - -const ignoredFiles = ['pages/404.mdx'] export type SearchSource = | MarkdownSource @@ -30,18 +26,9 @@ export type SearchSource = | LintWarningsGuideSource export async function fetchGuideSources() { - return ( - await Promise.all( - (await walk('content/guides')) - .filter( - ({ path }) => - /\.mdx?$/.test(path) && - !ignoredFiles.includes(path) && - !path.split(sep).some((part) => part.startsWith('_')) - ) - .map((entry) => new MarkdownLoader('guide', entry.path, { yaml: true }).load()) - ) - ).flat() + const guides = (await GuideModelLoader.allFromFs()).unwrapLeft() + + return guides.map((guide) => MarkdownLoader.fromGuideModel('guide', guide)) } export async function fetchOpenApiReferenceSource() { diff --git a/apps/docs/scripts/search/sources/markdown.ts b/apps/docs/scripts/search/sources/markdown.ts index 0c6f759d6e3..4127621b4bf 100644 --- a/apps/docs/scripts/search/sources/markdown.ts +++ b/apps/docs/scripts/search/sources/markdown.ts @@ -1,4 +1,4 @@ -import { SubsectionModel } from '../../../resources/guide/guideModel.js' +import { GuideModel, SubsectionModel } from '../../../resources/guide/guideModel.js' import { GuideModelLoader } from '../../../resources/guide/guideModelLoader.js' import { BaseLoader, BaseSource } from './base.js' @@ -32,6 +32,16 @@ export class MarkdownLoader extends BaseLoader { ), ] } + + static fromGuideModel(source: string, guide: GuideModel): MarkdownSource { + const path = guide.href ? guide.href.replace('https://supabase.com/docs', '') : '' + + return new MarkdownSource(source, path, guide.content ?? '', { + checksum: guide.checksum, + meta: guide.metadata, + sections: guide.subsections, + }) + } } export class MarkdownSource extends BaseSource {