From 274bd2f07091e368e67a87c146eacb4ed2efa812 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:00:12 -0400 Subject: [PATCH] feat(docs): add database lint warnings to search index (#36368) Adds support for indexing database lint warnings from the Splinter repository into the docs search. Creates a new LintWarningsGuideLoader that fetches markdown files from the Splinter repo and processes them as individual searchable entries in the database advisors guide. --- apps/docs/scripts/helpers.mdx.ts | 2 +- apps/docs/scripts/search/sources/index.ts | 16 ++- .../search/sources/lint-warnings-guide.ts | 116 ++++++++++++++++++ 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 apps/docs/scripts/search/sources/lint-warnings-guide.ts diff --git a/apps/docs/scripts/helpers.mdx.ts b/apps/docs/scripts/helpers.mdx.ts index cbcbfb8f709..c6d9ce672ef 100644 --- a/apps/docs/scripts/helpers.mdx.ts +++ b/apps/docs/scripts/helpers.mdx.ts @@ -1,4 +1,4 @@ -import { createHash } from 'crypto' +import { createHash } from 'node:crypto' import { ObjectExpression } from 'estree' import GithubSlugger from 'github-slugger' import matter from 'gray-matter' diff --git a/apps/docs/scripts/search/sources/index.ts b/apps/docs/scripts/search/sources/index.ts index 5f712ded87f..c46c46a7986 100644 --- a/apps/docs/scripts/search/sources/index.ts +++ b/apps/docs/scripts/search/sources/index.ts @@ -5,6 +5,7 @@ import { type GitHubDiscussionSource, fetchDiscussions, } from './github-discussion.js' +import { LintWarningsGuideLoader, type LintWarningsGuideSource } from './lint-warnings-guide.js' import { MarkdownLoader, type MarkdownSource } from './markdown.js' import { IntegrationLoader, type IntegrationSource, fetchPartners } from './partner-integrations.js' import { @@ -26,6 +27,7 @@ export type SearchSource = | CliReferenceSource | GitHubDiscussionSource | IntegrationSource + | LintWarningsGuideSource export async function fetchGuideSources() { return ( @@ -122,12 +124,23 @@ export async function fetchCliLibReferenceSource() { ).load() } +export async function fetchLintWarningsGuideSources() { + return new LintWarningsGuideLoader( + 'guide', + '/guides/database/database-advisors', + 'supabase', + 'splinter', + 'main', + 'docs' + ).load() +} + /** * Fetches all the sources we want to index for search */ export async function fetchAllSources() { const guideSources = fetchGuideSources() - + const lintWarningsGuideSources = fetchLintWarningsGuideSources() const openApiReferenceSource = fetchOpenApiReferenceSource() const jsLibReferenceSource = fetchJsLibReferenceSource() const dartLibReferenceSource = fetchDartLibReferenceSource() @@ -164,6 +177,7 @@ export async function fetchAllSources() { const sources: SearchSource[] = ( await Promise.all([ guideSources, + lintWarningsGuideSources, openApiReferenceSource, jsLibReferenceSource, dartLibReferenceSource, diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts new file mode 100644 index 00000000000..61d4f5e4e34 --- /dev/null +++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts @@ -0,0 +1,116 @@ +import { Octokit } from '@octokit/core' +import { BaseLoader, BaseSource } from './base.js' +import { createHash } from 'node:crypto' + +const getBasename = (path: string) => path.split('/').at(-1)!.replace(/\.md$/, '') + +export class LintWarningsGuideLoader extends BaseLoader { + type = 'markdown' as const + + constructor( + source: string, + path: string, + public org: string, + public repo: string, + public branch: string, + public docsDir: string + ) { + super(source, path) + } + + async load() { + const octokit = new Octokit() + + const response = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', { + owner: this.org, + repo: this.repo, + path: this.docsDir, + ref: this.branch, + headers: { + 'X-GitHub-Api-Version': '2022-11-28', + }, + }) + + if (response.status >= 400) { + throw Error(`Could not get contents of repo ${this.org}/${this.repo}`) + } + + if (!Array.isArray(response.data)) { + throw Error( + 'Reading a directory, not a file. Should not reach this, solely to appease Typescript.' + ) + } + + const lintsList = response.data.filter(({ path }) => /docs\/\d+.+\.md$/.test(path)) + + // Fetch all lint files and combine them into a single guide + const lints = await Promise.all( + lintsList.map(async ({ path }) => { + const fileResponse = await fetch( + `https://raw.githubusercontent.com/${this.org}/${this.repo}/${this.branch}/${path}` + ) + + if (fileResponse.status >= 400) { + throw Error(`Could not get contents of file ${this.org}/${this.repo}/${path}`) + } + + const content = await fileResponse.text() + const basename = getBasename(path) + + return { + path: basename, + content, + originalPath: path, + } + }) + ) + + // Create a separate source for each lint file + return lints.map( + (lint) => + new LintWarningsGuideSource( + this.source, + `${this.path}?queryGroups=lint&lint=${lint.path}`, + lint + ) + ) + } +} + +export class LintWarningsGuideSource extends BaseSource { + type = 'markdown' as const + + constructor( + source: string, + path: string, + public lint: { + path: string + content: string + originalPath: string + } + ) { + super(source, path) + } + + process() { + this.checksum = createHash('sha256').update(this.lint.content).digest('base64') + + this.meta = { + title: `Database Advisor: Lint ${this.lint.path}`, + } + + this.sections = [ + { + content: this.lint.content, + }, + ] + + return { checksum: this.checksum, meta: this.meta, sections: this.sections } + } + + extractIndexedContent(): string { + const sections = this.sections ?? [] + const sectionText = sections.map(({ content }) => content).join('\n\n') + return `# ${this.lint.path}\n\n${sectionText}` + } +}