From 4a82aabaf020ec8b58d12ba0f3d148d15b1b5dca Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:58:46 -0400 Subject: [PATCH] feat(docs): sync troubleshooting (#29933) When a troubleshooting entry is created or updated in this repo, sync it to GitHub. This creates a new troubleshooting discussion if one does not exist, and updates the content on an existing troubleshooting discussion if it has changed. To preserve the connections between GitHub discussions and Markdown files, record the discussion ID and URL to the database. The database ID is then written to the Markdown file, and a new PR is generated to commit the new Markdown with ID. There is currently no mechanism to delete outdated discussions. That will be a follow-up. --- .github/CODEOWNERS | 3 + .../workflows/docs-sync-troubleshooting.yml | 51 ++ .../guides/troubleshooting/[slug]/page.tsx | 4 + apps/docs/app/guides/troubleshooting/page.tsx | 3 + ...monitor-supavisor-postgres-connections.mdx | 15 +- .../supabase-grafana-memory-charts.mdx | 13 +- .../docs/features/docs/Reference.sections.tsx | 2 +- .../features/docs/Troubleshooting.page.tsx | 12 +- .../features/docs/Troubleshooting.script.mjs | 439 ++++++++++++++++++ .../docs/features/docs/Troubleshooting.ui.tsx | 19 +- .../docs/Troubleshooting.utils.common.mjs | 185 ++++++++ .../features/docs/Troubleshooting.utils.ts | 164 ++----- apps/docs/lib/supabaseAdmin.ts | 18 + apps/docs/package.json | 2 + package-lock.json | 13 + packages/common/database-types.ts | 39 ++ 16 files changed, 829 insertions(+), 153 deletions(-) create mode 100644 .github/workflows/docs-sync-troubleshooting.yml create mode 100644 apps/docs/features/docs/Troubleshooting.script.mjs create mode 100644 apps/docs/features/docs/Troubleshooting.utils.common.mjs create mode 100644 apps/docs/lib/supabaseAdmin.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b7074fd4b1a..40167c5036e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,9 @@ /apps/studio/ @supabase/Dashboard +# temporary until we've ironed out all the problems with sync workflow +/apps/docs/content/troubleshooting/ @charislam + /apps/www/ @supabase/marketing /apps/www/public/images/blog @supabase/marketing /apps/www/lib/redirects.js diff --git a/.github/workflows/docs-sync-troubleshooting.yml b/.github/workflows/docs-sync-troubleshooting.yml new file mode 100644 index 00000000000..0b2d3d04e86 --- /dev/null +++ b/.github/workflows/docs-sync-troubleshooting.yml @@ -0,0 +1,51 @@ +name: '[Docs] Sync troubleshooting guides to GitHub Discussions' + +on: + push: + branches: + - master + paths: + - 'apps/docs/content/troubleshooting/**' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-troubleshooting: + runs-on: ubuntu-latest + + env: + SEARCH_GITHUB_APP_ID: ${{ secrets.SEARCH_GITHUB_APP_ID }} + SEARCH_GITHUB_APP_INSTALLATION_ID: ${{ secrets.SEARCH_GITHUB_APP_INSTALLATION_ID }} + SEARCH_GITHUB_APP_PRIVATE_KEY: ${{ secrets.SEARCH_GITHUB_APP_PRIVATE_KEY }} + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }} + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + apps/docs + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run Troubleshooting script + run: npm --prefix=apps/docs run troubleshooting:sync + + - name: Create Pull Request + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 + with: + commit-message: '[bot] sync troubleshooting guides to db' + title: '[bot] sync troubleshooting guides to db' + author: 'github-docs-sync-bot ' + branch: 'bot/docs-sync-troubleshooting' + branch-suffix: 'random' diff --git a/apps/docs/app/guides/troubleshooting/[slug]/page.tsx b/apps/docs/app/guides/troubleshooting/[slug]/page.tsx index 428a2e84adc..01d40abeef7 100644 --- a/apps/docs/app/guides/troubleshooting/[slug]/page.tsx +++ b/apps/docs/app/guides/troubleshooting/[slug]/page.tsx @@ -4,6 +4,10 @@ import TroubleshootingPage from '~/features/docs/Troubleshooting.page' import { getAllTroubleshootingEntries, getArticleSlug } from '~/features/docs/Troubleshooting.utils' import { PROD_URL } from '~/lib/constants' +// 60 seconds/minute * 60 minutes/hour * 24 hours/day +export const revalidate = 86_400 +export const dynamicParams = false + export default async function TroubleshootingEntryPage({ params: { slug }, }: { diff --git a/apps/docs/app/guides/troubleshooting/page.tsx b/apps/docs/app/guides/troubleshooting/page.tsx index 587e84fa1ae..ac0be072bfe 100644 --- a/apps/docs/app/guides/troubleshooting/page.tsx +++ b/apps/docs/app/guides/troubleshooting/page.tsx @@ -16,6 +16,9 @@ import { TROUBLESHOOTING_CONTAINER_ID } from '~/features/docs/Troubleshooting.ut import { SidebarSkeleton } from '~/layouts/MainSkeleton' import { PROD_URL } from '~/lib/constants' +// 60 seconds/minute * 60 minutes/hour * 24 hours/day +export const revalidate = 86_400 + export default async function GlobalTroubleshootingPage() { const troubleshootingEntries = await getAllTroubleshootingEntries() const keywords = await getAllTroubleshootingKeywords() diff --git a/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx b/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx index 57662e00c61..1d1e733344e 100644 --- a/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx +++ b/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx @@ -1,15 +1,10 @@ --- title = "How to monitor Postgres and Supavisor connections" -topics = [ -"database", -"supavisor", -] -keywords = [ -"Grafana", -"connections", -"performance", -"pooler", -] +github_url = "https://github.com/orgs/supabase/discussions/27141" +topics = [ "database", "supavisor" ] +keywords = [ "Grafana", "connections", "performance", "pooler" ] +date_created = "2024-06-09" +database_id = "bdd39473-ae02-41f5-b75d-877a79f361f6" --- This guide explains how connections impact your Supabase database's performance and how to optimize them for better resource utilization. diff --git a/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx b/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx index ba1c5dcd9cc..ccb98410f56 100644 --- a/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx +++ b/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx @@ -1,13 +1,10 @@ --- title = "Interpreting Supabase Grafana Memory Charts" -topics = [ -"database", -] -keywords = [ -"Grafana", -"performance", -"memory" -] +github_url = "https://github.com/orgs/supabase/discussions/27021" +topics = [ "database" ] +keywords = [ "Grafana", "performance", "memory" ] +date_created = "2024-06-05" +database_id = "179d70f3-1e26-4346-9ee8-d340fad382a3" --- > [Supabase Grafana Installation Guide](/docs/guides/platform/metrics#deploying-supabase-grafana) diff --git a/apps/docs/features/docs/Reference.sections.tsx b/apps/docs/features/docs/Reference.sections.tsx index 6bab6c1acc3..79456afd561 100644 --- a/apps/docs/features/docs/Reference.sections.tsx +++ b/apps/docs/features/docs/Reference.sections.tsx @@ -298,7 +298,7 @@ async function ApiEndpointSection({ link, section, servicePath }: ApiEndpointSec
- + {endpointDetails.method} {endpointDetails.path} diff --git a/apps/docs/features/docs/Troubleshooting.page.tsx b/apps/docs/features/docs/Troubleshooting.page.tsx index 5acc242328c..59bfb3d4152 100644 --- a/apps/docs/features/docs/Troubleshooting.page.tsx +++ b/apps/docs/features/docs/Troubleshooting.page.tsx @@ -4,11 +4,15 @@ import Breadcrumbs from '~/components/Breadcrumbs' import { Feedback } from '~/components/Feedback' import { SidebarSkeleton } from '~/layouts/MainSkeleton' import { MDXRemoteBase } from './MdxBase' -import { type ITroubleshootingEntry } from './Troubleshooting.utils' +import { getTroubleshootingUpdatedDates, type ITroubleshootingEntry } from './Troubleshooting.utils' import Link from 'next/link' import { formatError, serializeTroubleshootingSearchParams } from './Troubleshooting.utils.shared' -export default function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { +export default async function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + return (

{entry.data.title}

- {entry.data.updated_at && ( + {dateUpdated && (

- Last edited: {entry.data.updated_at.toLocaleString()} + Last edited: {dateUpdated.toLocaleDateString()}

)}
diff --git a/apps/docs/features/docs/Troubleshooting.script.mjs b/apps/docs/features/docs/Troubleshooting.script.mjs new file mode 100644 index 00000000000..932def8cc23 --- /dev/null +++ b/apps/docs/features/docs/Troubleshooting.script.mjs @@ -0,0 +1,439 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +/** + * Sync new troubleshooting entries from the GitHub repo with GitHub + * Discussions. + */ + +import { createAppAuth } from '@octokit/auth-app' +import { Octokit } from '@octokit/core' +import { createClient } from '@supabase/supabase-js' +import { createHash } from 'crypto' +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { readFile, writeFile } from 'node:fs/promises' +import { stringify } from 'smol-toml' +import toml from 'toml' + +import { + getAllTroubleshootingEntriesInternal as getAllTroubleshootingEntries, + getArticleSlug, +} from './Troubleshooting.utils.common.mjs' + +import 'dotenv/config' + +const REPOSITORY_ID = 'MDEwOlJlcG9zaXRvcnkyMTQ1ODcxOTM=' +const TROUBLESHOOTING_CATEGORY_ID = 'DIC_kwDODMpXOc4CUvEr' +const REPOSITORY_OWNER = 'supabase' +const REPOSITORY_NAME = 'supabase' + +/** + * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingEntry} TroubleshootingEntry + * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingMetadata} TroubleshootingMetadata + */ + +/** + * @type {import('@octokit/core').Octokit} + */ +let octokitInstance +/** + * @type {SupabaseClient} + */ +let supabaseAdminClient + +function octokit() { + if (!octokitInstance) { + octokitInstance = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: process.env.SEARCH_GITHUB_APP_ID, + installationId: process.env.SEARCH_GITHUB_APP_INSTALLATION_ID, + privateKey: process.env.SEARCH_GITHUB_APP_PRIVATE_KEY, + }, + }) + } + + return octokitInstance +} + +export function supabaseAdmin() { + if (!supabaseAdminClient) { + supabaseAdminClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SECRET_KEY + ) + } + + return supabaseAdminClient +} + +async function syncTroubleshootingEntries() { + console.log('[START] Syncing troubleshooting entries to GitHub') + + const [troubleshootingEntries, discussions] = await Promise.all([ + getAllTroubleshootingEntries(), + getAllTroubleshootingDiscussions(), + ]) + + console.log(`[INFO] Found ${troubleshootingEntries.length} existing entries`) + + const tasks = troubleshootingEntries.map(async (entry) => { + const databaseId = entry.data.database_id + if (databaseId.startsWith('pseudo-')) { + // The database entry is faked, so we may need to create a new one. + // There's also an edge case we need to check for: the entry has already + // been created, but the new database ID hasn't been written to the file + // yet. + if (await entryExists(entry)) return + + const discussion = entry.data.github_url + ? await getGithubIdForDiscussion(discussions, entry) + : await createGithubDiscussion(entry) + + let id + try { + id = await insertNewTroubleshootingEntry(entry, discussion) + } catch (error) { + console.error(`[ERROR] Failed to insert new entry for ${entry.data.title}: %O`, error) + console.log( + `[INFO] Rolling back discussion creation for ${entry.data.title} (GitHub ID ${discussion.id})` + ) + await rollbackGithubDiscussion(discussion.id) + throw error + } + + await updateFileId(entry, id) + } else { + // The database entry already exists, so check for updates. + const contentHasChanged = await updateChecksumIfNeeded(entry) + if (contentHasChanged) { + await updateGithubDiscussion(entry) + } + } + }) + + const results = await Promise.allSettled(tasks) + let hasErrors = false + results.forEach((result, index) => { + if (result.status === 'rejected') { + console.error( + `[ERROR] Failed to insert and/or update for ${troubleshootingEntries[index].filePath}:\n%O`, + result.reason + ) + hasErrors = true + } + }) + + return hasErrors +} + +/** + * @param {TroubleshootingEntry} entry + * @throws If the database check fails for some unknown reason + */ +async function entryExists(entry) { + const checksum = calculateChecksum(entry.content) + const { error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('id') + .eq('checksum', checksum) + .single() + + if (error) { + if (error.code === 'PGRST116') { + // No entry found + return false + } + throw error + } + + console.log( + `[INFO] Entry for ${entry.data.title} already exists. Not creating a new one to prevent duplicates.` + ) + return true +} + +/** + * @param {string} content + */ +function calculateChecksum(content) { + // Normalize to ignore changes that don't affect the final displayed content. + const mdast = fromMarkdown(content, { + extensions: [gfm(), mdxjs()], + mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()], + }) + const normalized = toMarkdown(mdast, { extensions: [gfmToMarkdown(), mdxToMarkdown()] }) + + return createHash('sha256').update(normalized).digest('base64') +} + +/** + * @param {TroubleshootingEntry} entry + * + * @param {Object} discussion + * @param {string} discussion.id + * @param {string} discussion.url + * + * @throws If the database insertion fails + */ +async function insertNewTroubleshootingEntry(entry, discussion) { + console.log(`[INFO] Inserting entry for ${entry.data.title} into DB`) + + const now = new Date().toISOString() + const checksum = calculateChecksum(entry.content) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .insert({ + api: entry.data.api, + checksum, + date_created: entry.data.date_created?.toISOString() ?? now, + date_updated: now, + errors: entry.data.errors, + github_id: discussion.id, + github_url: discussion.url, + keywords: entry.data.keywords, + title: entry.data.title, + topics: entry.data.topics, + }) + .select('id') + .single() + if (error) { + throw error + } + + return data.id +} + +/** + * @param {TroubleshootingEntry} entry + * + * @throws If the database update fails + */ +async function updateChecksumIfNeeded(entry) { + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('checksum') + .eq('id', entry.data.database_id) + .single() + if (error) { + throw error + } + + const newChecksum = calculateChecksum(entry.content) + if (data.checksum !== newChecksum) { + console.log(`[INFO] Content changed for ${entry.data.title}. Updating checksum.`) + + const now = new Date().toISOString() + const { error } = await supabaseAdmin() + .from('troubleshooting_entries') + .update({ + checksum: newChecksum, + date_updated: now, + }) + .eq('id', entry.data.database_id) + + if (error) { + throw error + } + + return true + } + + return false +} + +/** + * @param {TroubleshootingEntry} entry + */ +function addCanonicalUrl(entry) { + const docsUrl = 'https://supabase.com/docs/guides/troubleshooting/' + getArticleSlug(entry.data) + const content = + `_This is a copy of a troubleshooting article on Supabase's docs site. It may be missing some details from the original. View the [original article](${docsUrl})._\n\n` + + entry.contentWithoutJsx + return content +} + +/** + * @param {TroubleshootingEntry} entry + */ +async function createGithubDiscussion(entry) { + console.log(`[INFO] Creating GitHub discussion for ${entry.data.title}`) + const content = addCanonicalUrl(entry) + + const mutation = ` + mutation { + createDiscussion(input: { + repositoryId: "${REPOSITORY_ID}", + categoryId: "${TROUBLESHOOTING_CATEGORY_ID}", + body: "${content}", + title: "${entry.data.title}" + }) { + discussion { + id + url + } + } + } + ` + + const { + createDiscussion: { discussion }, + } = await octokit().graphql(mutation) + console.log(`[INFO] Created GitHub discussion for ${entry.data.title}: %s`, discussion.url) + return discussion +} + +/** + * @returns {Promise<{id: string, url: string}[]>} + */ +async function getAllTroubleshootingDiscussions() { + const query = ` + query getDiscussions($cursor: String) { + repository(owner: "${REPOSITORY_OWNER}", name: "${REPOSITORY_NAME}") { + discussions(first: 100, after: $cursor, categoryId: "${TROUBLESHOOTING_CATEGORY_ID}") { + pageInfo { + hasNextPage + endCursor + } + nodes { + id + url + } + } + } + } + ` + + const discussions = [] + let hasNextPage = true + let cursor + + while (hasNextPage) { + const { + repository: { + discussions: { nodes: moreDiscussions, pageInfo }, + }, + } = await octokit().graphql(query, { cursor }) + + discussions.push(...moreDiscussions) + hasNextPage = pageInfo.hasNextPage + cursor = pageInfo.endCursor + } + + return discussions +} + +/** + * @param {{id: string, url: string}[]} discussions + * @param {TroubleshootingEntry} entry + * + * @throws If matching discussion not found + */ +async function getGithubIdForDiscussion(discussions, entry) { + const matchingDiscussion = discussions.find( + (discussion) => discussion.url === entry.data.github_url + ) + if (!matchingDiscussion) { + throw new Error(`No matching discussion found for URL: ${entry.data.github_url}`) + } + return matchingDiscussion +} + +/** + * @param {TroubleshootingEntry} entry + * + * @throws If stored discussion ID for entry not found + */ +async function updateGithubDiscussion(entry) { + console.log(`[INFO] Updating discussion content for ${entry.data.title}`) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('github_id') + .eq('id', entry.data.database_id) + .single() + if (error) { + throw error + } + + const content = addCanonicalUrl(entry) + const mutation = ` + mutation { + updateDiscussion(input: { + discussionId: "${data.github_id}", + body: "${content}", + }) { + discussion { + id + } + } + } + ` + + await octokit().graphql(mutation) + console.log(`[INFO] Updated discussion content for ${entry.data.title}`) +} + +/** @param {string} id */ +async function rollbackGithubDiscussion(id) { + try { + const mutation = ` + mutation { + deleteDiscussion(input: { + id: "${id}", + }) { + discussion { + id + } + } + } + ` + + await octokit().graphql(mutation) + console.log(`[INFO] Rolled back discussion creation for ${id}`) + } catch (error) { + console.error(`[ERROR] Failed to rollback discussion creation for ${id}: %O`, error) + } +} + +/** + * @param {TroubleshootingEntry} entry + * @param {string} id + * + * @throws Passes through readFile and writeFile errors without catching + */ +async function updateFileId(entry, id) { + console.log(`[INFO] Writing database ID to file for ${entry.filePath}`) + + const fileContents = await readFile(entry.filePath, 'utf-8') + const { data, content } = matter(fileContents, { + language: 'toml', + engines: { toml: toml.parse.bind(toml) }, + }) + data.database_id = id + + const newFrontmatter = stringify(data) + const newContent = `---\n${newFrontmatter}\n---\n${content}` + + await writeFile(entry.filePath, newContent) +} + +async function main() { + try { + const hasErrors = await syncTroubleshootingEntries() + if (hasErrors) { + process.exit(1) + } + } catch (error) { + console.error(`[ERROR] %O`, error) + process.exit(1) + } +} + +main() diff --git a/apps/docs/features/docs/Troubleshooting.ui.tsx b/apps/docs/features/docs/Troubleshooting.ui.tsx index 06ca618cff2..b3fec96bff8 100644 --- a/apps/docs/features/docs/Troubleshooting.ui.tsx +++ b/apps/docs/features/docs/Troubleshooting.ui.tsx @@ -2,12 +2,20 @@ import { Wrench } from 'lucide-react' import Link from 'next/link' import { type PropsWithChildren, useCallback } from 'react' -import { type ITroubleshootingEntry, getArticleSlug } from './Troubleshooting.utils' +import { + type ITroubleshootingEntry, + getArticleSlug, + getTroubleshootingUpdatedDates, +} from './Troubleshooting.utils' import { TroubleshootingFilter } from './Troubleshooting.ui.client' import { formatError, TROUBLESHOOTING_DATA_ATTRIBUTES } from './Troubleshooting.utils.shared' import { cn } from 'ui' -export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { +export async function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + const keywords = [...entry.data.topics, ...(entry.data.keywords ?? [])] const attributes = { [TROUBLESHOOTING_DATA_ATTRIBUTES.QUERY_ATTRIBUTE]: @@ -58,14 +66,13 @@ export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry ))}
- {entry.data.updated_at && + {dateUpdated && (() => { - const date = new Date(entry.data.updated_at) const options = { month: 'short', day: 'numeric' } as Intl.DateTimeFormatOptions - if (date.getFullYear() !== new Date().getFullYear()) { + if (dateUpdated.getFullYear() !== new Date().getFullYear()) { options.year = 'numeric' } - return date.toLocaleDateString(undefined, options) + return dateUpdated.toLocaleDateString(undefined, options) })()}
diff --git a/apps/docs/features/docs/Troubleshooting.utils.common.mjs b/apps/docs/features/docs/Troubleshooting.utils.common.mjs new file mode 100644 index 00000000000..e2571f07819 --- /dev/null +++ b/apps/docs/features/docs/Troubleshooting.utils.common.mjs @@ -0,0 +1,185 @@ +/** + * This file is for utils needed in both the Next.js app build and the + * troubleshooting sync script. Because of unsolved problems with imports, the + * script is a mjs file instead of a ts file. Any dependencies that are needed + * in both places are defined here, and then typed in Troubleshooting.utils.ts + * as required. + */ + +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { readdir, readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import toml from 'toml' +import { visit } from 'unist-util-visit' +import { v4 as uuidv4 } from 'uuid' +import { z } from 'zod' + +export const TROUBLESHOOTING_DIRECTORY = join(process.cwd(), 'content/troubleshooting') + +/** + * @typedef {Object} TroubleshootingEntry + * @property {string} filePath + * @property {string} content + * @property {string} contentWithoutJsx + * @property {TroubleshootingMetadata} data + */ + +/** + * @typedef {Object} TroubleshootingMetadata + * @property {string} title + * @property {string[]} topics + * @property {string[]} [keywords] + * @property {Object} [api] + * @property {string[]} [api.sdk] + * @property {string[]} [api.management_api] + * @property {string[]} [api.cli] + * @property {AssociatedError[]} [errors] + * @property {string} database_id + * @property {string} [github_url] + * @property {Date} [date_created] + */ + +/** + * @typedef {Object} AssociatedError + * @property {number} [http_status_code] + * @property {string} [code] + * @property {string} [message] + */ + +export const TroubleshootingSchema = z + .object({ + title: z.string(), + topics: z.array( + z.enum([ + 'ai', + 'auth', + 'branching', + 'cli', + 'database', + 'functions', + 'platform', + 'realtime', + 'self-hosting', + 'storage', + 'studio', + 'supavisor', + 'terraform', + ]) + ), + keywords: z.array(z.string()).optional(), + api: z + .object({ + sdk: z.array(z.string()).optional(), + management_api: z.array(z.string()).optional(), + cli: z.array(z.string()).optional(), + }) + .strict() + .optional(), + errors: z + .array( + z + .object({ + http_status_code: z.number().optional(), + code: z.string().optional(), + message: z.string().optional(), + }) + .strict() + ) + .optional(), + database_id: z.string().default(`pseudo-${uuidv4()}`), + github_url: z.string().url().optional(), + date_created: z.date({ coerce: true }).optional(), + }) + .strict() + +/* + * @param {unknown} troubleshootingMetadata + */ +function validateTroubleshootingMetadata(troubleshootingMetadata) { + return TroubleshootingSchema.safeParse(troubleshootingMetadata) +} + +/* + * @returns {Promise} + */ +export async function getAllTroubleshootingEntriesInternal() { + const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, { + recursive: true, + }) + const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => { + const isHidden = entry.startsWith('_') + if (isHidden) return null + + const filePath = join(TROUBLESHOOTING_DIRECTORY, entry) + + const isFile = (await stat(filePath)).isFile() + if (!isFile) return null + + const fileContents = await readFile(filePath, 'utf-8') + const { content, data: frontmatter } = matter(fileContents, { + language: 'toml', + engines: { toml: toml.parse.bind(toml) }, + }) + + const parseResult = validateTroubleshootingMetadata(frontmatter) + if ('error' in parseResult) { + console.error( + `Error validating troubleshooting metadata\nEntry:%O\nError:%O`, + frontmatter, + parseResult.error + ) + return null + } + + const mdxTree = fromMarkdown(content, { + extensions: [gfm(), mdxjs()], + mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()], + }) + visit(mdxTree, (node) => { + if ('children' in node) { + node.children = node.children.filter( + (child) => + ![ + 'mdxJsxFlowExpression', + 'mdxJsxTextExpression', + 'mdxFlowExpression', + 'mdxTextExpression', + 'mdxJsxFlowElement', + 'mdxJsxTextElement', + 'mdxJsxExpressionAttribute', + 'mdxJsxAttribute', + 'mdxJsxAttributeValueExpression', + 'mdxjsEsm', + ].includes(child.type) + ) + } + }) + const contentWithoutJsx = toMarkdown(mdxTree, { + extensions: [gfmToMarkdown()], + }) + + return { + filePath, + content, + contentWithoutJsx, + data: parseResult.data, + } + }) + + return (await Promise.all(troubleshootingFiles)).filter(Boolean) +} + +/** + * @param {TroubleshootingMetadata} meta + */ +export function getArticleSlug(meta) { + const slugifiedTitle = meta.title.toLowerCase().replace(/\s+/g, '-') + const escapedTitle = encodeURIComponent(slugifiedTitle) + return escapedTitle +} diff --git a/apps/docs/features/docs/Troubleshooting.utils.ts b/apps/docs/features/docs/Troubleshooting.utils.ts index 0b2c755303a..2f858fcd9dd 100644 --- a/apps/docs/features/docs/Troubleshooting.utils.ts +++ b/apps/docs/features/docs/Troubleshooting.utils.ts @@ -1,137 +1,39 @@ -import matter from 'gray-matter' -import { fromMarkdown } from 'mdast-util-from-markdown' -import { mdxFromMarkdown } from 'mdast-util-mdx' -import { toMarkdown } from 'mdast-util-to-markdown' -import { mdxjs } from 'micromark-extension-mdxjs' -import { readdir, readFile, stat } from 'node:fs/promises' -import { join } from 'node:path' -import toml from 'toml' -import { visit } from 'unist-util-visit' -import { v4 as uuidv4 } from 'uuid' +import { cache } from 'react' import { z } from 'zod' -import { DOCS_DIRECTORY } from 'lib/docs' import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs' +import { supabaseAdmin } from '~/lib/supabaseAdmin' +import { + getAllTroubleshootingEntriesInternal, + getArticleSlug as getArticleSlugInternal, + TroubleshootingSchema, + TROUBLESHOOTING_DIRECTORY, +} from './Troubleshooting.utils.common.mjs' import { formatError } from './Troubleshooting.utils.shared' -const TROUBLESHOOTING_DIRECTORY = join(DOCS_DIRECTORY, 'content/troubleshooting') +// We don't have strictNullChecks on because we still have too many violations +// in the code base, so required types aren't typed properly by Zod. +export type ITroubleshootingMetadata = z.infer & + Pick>, 'title' | 'topics' | 'database_id'> -const TroubleshootingSchema = z - .object({ - title: z.string(), - topics: z.array( - z.enum([ - 'ai', - 'auth', - 'branching', - 'cli', - 'database', - 'functions', - 'platform', - 'realtime', - 'self-hosting', - 'storage', - 'studio', - 'supavisor', - 'terraform', - ]) - ), - keywords: z.array(z.string()).optional(), - api: z - .object({ - sdk: z.array(z.string()).optional(), - management_api: z.array(z.string()).optional(), - cli: z.array(z.string()).optional(), - }) - .strict() - .optional(), - errors: z - .array( - z - .object({ - http_status_code: z.number().optional(), - code: z.string().optional(), - message: z.string().optional(), - }) - .strict() - ) - .optional(), - database_id: z.string().uuid().default(uuidv4()), - created_at: z.date({ coerce: true }).optional(), - updated_at: z.date({ coerce: true }).optional(), - }) - .strict() - -export type ITroubleshootingMetadata = z.infer - -const validateTroubleshootingMetadata = (troubleshootingMetadata: unknown) => { - return TroubleshootingSchema.safeParse(troubleshootingMetadata) +export interface ITroubleshootingEntry { + filePath: string + content: string + contentWithoutJsx: string + data: ITroubleshootingMetadata } -async function getAllTroubleshootingEntriesInternal() { - const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, { - recursive: true, - }) - const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => { - const isHidden = entry.startsWith('_') - if (isHidden) return null +export const getArticleSlug = getArticleSlugInternal - const isFile = (await stat(join(TROUBLESHOOTING_DIRECTORY, entry))).isFile() - if (!isFile) return null - - const fileContents = await readFile(join(TROUBLESHOOTING_DIRECTORY, entry), 'utf-8') - const { content, data: frontmatter } = matter(fileContents, { - language: 'toml', - engines: { toml: toml.parse.bind(toml) }, - }) - - const parseResult = validateTroubleshootingMetadata(frontmatter) - if ('error' in parseResult) { - console.error(`Error validating troubleshooting metadata for ${entry}:`, parseResult.error) - return null - } - - const mdxTree = fromMarkdown(content, { - extensions: [mdxjs()], - mdastExtensions: [mdxFromMarkdown()], - }) - visit(mdxTree, (node) => { - if ('children' in node) { - node.children = node.children.filter( - (child) => - ![ - 'mdxJsxFlowExpression', - 'mdxJsxTextExpression', - 'mdxFlowExpression', - 'mdxTextExpression', - 'mdxJsxFlowElement', - 'mdxJsxTextElement', - 'mdxJsxExpressionAttribute', - 'mdxJsxAttribute', - 'mdxJsxAttributeValueExpression', - 'mdxjsEsm', - ].includes(child.type) - ) - } - }) - const contentWithoutJsx = toMarkdown(mdxTree) - - return { - content, - contentWithoutJsx, - data: parseResult.data, - } - }) - - return (await Promise.all(troubleshootingFiles)).filter(Boolean) +async function getAllTroubleshootingEntriesTyped() { + const result = (await getAllTroubleshootingEntriesInternal()) as ITroubleshootingEntry[] + return result } - export const getAllTroubleshootingEntries = cache_fullProcess_withDevCacheBust( - getAllTroubleshootingEntriesInternal, + getAllTroubleshootingEntriesTyped, TROUBLESHOOTING_DIRECTORY, () => JSON.stringify([]) ) -export type ITroubleshootingEntry = Awaited>[number] export async function getAllTroubleshootingKeywords() { const entries = await getAllTroubleshootingEntries() @@ -185,8 +87,22 @@ export async function getAllTroubleshootingErrors() { return Array.from(allErrors).sort(sortErrors) } -export function getArticleSlug(entry: ITroubleshootingMetadata) { - const slugifiedTitle = entry.title.toLowerCase().replace(/\s+/g, '-') - const escapedTitle = encodeURIComponent(slugifiedTitle) - return escapedTitle +async function getTroubleshootingUpdatedDatesInternal() { + const databaseIds = (await getAllTroubleshootingEntries()) + .map((entry) => entry.data.database_id) + .filter((id) => !id.startsWith('pseudo-')) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('id, date_updated') + .in('id', databaseIds) + if (error) { + console.error(error) + } + + return (data ?? []).reduce((acc, entry) => { + acc.set(entry.id, new Date(entry.date_updated)) + return acc + }, new Map()) } +export const getTroubleshootingUpdatedDates = cache(getTroubleshootingUpdatedDatesInternal) diff --git a/apps/docs/lib/supabaseAdmin.ts b/apps/docs/lib/supabaseAdmin.ts new file mode 100644 index 00000000000..2c58ea65395 --- /dev/null +++ b/apps/docs/lib/supabaseAdmin.ts @@ -0,0 +1,18 @@ +import 'server-only' + +import { createClient, type SupabaseClient } from '@supabase/supabase-js' + +import { type Database } from 'common' + +let supabaseAdminClient: SupabaseClient | null = null + +export function supabaseAdmin() { + if (!supabaseAdminClient) { + supabaseAdminClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SECRET_KEY + ) + } + + return supabaseAdminClient +} diff --git a/apps/docs/package.json b/apps/docs/package.json index 62f16690760..9cf97308475 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -18,6 +18,7 @@ "test:smoke": "npm run codegen:references && vitest -t \"prod smoke test\"", "embeddings": "tsx scripts/search/generate-embeddings.ts", "embeddings:refresh": "npm run embeddings -- --refresh", + "troubleshooting:sync": "node features/docs/Troubleshooting.script.mjs", "last-changed": "tsx scripts/last-changed.ts", "last-changed:reset": "npm run last-changed -- --reset", "codegen:references": "tsx features/docs/Reference.generated.script.ts", @@ -127,6 +128,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", diff --git a/package-lock.json b/package-lock.json index 2a1db90666b..2772cec9bae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -958,6 +958,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", @@ -39644,6 +39645,18 @@ "npm": ">= 3.0.0" } }, + "node_modules/smol-toml": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.0.tgz", + "integrity": "sha512-tWpi2TsODPScmi48b/OQZGi2lgUmBCHy6SZrhi/FdnnHiU1GwebbCfuQuxsC3nHaLwtYeJGPrDZDIeodDOc4pA==", + "dev": true, + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, "node_modules/snake-case": { "version": "3.0.4", "dev": true, diff --git a/packages/common/database-types.ts b/packages/common/database-types.ts index c13e9a2de83..79ee7683f14 100644 --- a/packages/common/database-types.ts +++ b/packages/common/database-types.ts @@ -318,9 +318,11 @@ export type Database = { troubleshooting_entries: { Row: { api: Json | null + checksum: string date_created: string date_updated: string errors: Json[] | null + github_id: string github_url: string id: string keywords: string[] | null @@ -329,9 +331,11 @@ export type Database = { } Insert: { api?: Json | null + checksum: string date_created?: string date_updated?: string errors?: Json[] | null + github_id: string github_url: string id?: string keywords?: string[] | null @@ -340,9 +344,11 @@ export type Database = { } Update: { api?: Json | null + checksum?: string date_created?: string date_updated?: string errors?: Json[] | null + github_id?: string github_url?: string id?: string keywords?: string[] | null @@ -464,6 +470,33 @@ export type Database = { } Returns: unknown } + json_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonb_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonschema_is_valid: { + Args: { + schema: Json + } + Returns: boolean + } + jsonschema_validation_errors: { + Args: { + schema: Json + instance: Json + } + Returns: string[] + } match_page_sections_v2: { Args: { embedding: string @@ -491,6 +524,12 @@ export type Database = { } Returns: string } + validate_troubleshooting_errors: { + Args: { + errors: Json[] + } + Returns: boolean + } vector_avg: { Args: { "": number[]