diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b7074fd4b1a..40167c5036e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,9 @@ /apps/studio/ @supabase/Dashboard +# temporary until we've ironed out all the problems with sync workflow +/apps/docs/content/troubleshooting/ @charislam + /apps/www/ @supabase/marketing /apps/www/public/images/blog @supabase/marketing /apps/www/lib/redirects.js diff --git a/.github/workflows/docs-sync-troubleshooting.yml b/.github/workflows/docs-sync-troubleshooting.yml new file mode 100644 index 00000000000..0b2d3d04e86 --- /dev/null +++ b/.github/workflows/docs-sync-troubleshooting.yml @@ -0,0 +1,51 @@ +name: '[Docs] Sync troubleshooting guides to GitHub Discussions' + +on: + push: + branches: + - master + paths: + - 'apps/docs/content/troubleshooting/**' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-troubleshooting: + runs-on: ubuntu-latest + + env: + SEARCH_GITHUB_APP_ID: ${{ secrets.SEARCH_GITHUB_APP_ID }} + SEARCH_GITHUB_APP_INSTALLATION_ID: ${{ secrets.SEARCH_GITHUB_APP_INSTALLATION_ID }} + SEARCH_GITHUB_APP_PRIVATE_KEY: ${{ secrets.SEARCH_GITHUB_APP_PRIVATE_KEY }} + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }} + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + apps/docs + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run Troubleshooting script + run: npm --prefix=apps/docs run troubleshooting:sync + + - name: Create Pull Request + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 + with: + commit-message: '[bot] sync troubleshooting guides to db' + title: '[bot] sync troubleshooting guides to db' + author: 'github-docs-sync-bot ' + branch: 'bot/docs-sync-troubleshooting' + branch-suffix: 'random' diff --git a/apps/docs/app/guides/troubleshooting/[slug]/page.tsx b/apps/docs/app/guides/troubleshooting/[slug]/page.tsx index 428a2e84adc..01d40abeef7 100644 --- a/apps/docs/app/guides/troubleshooting/[slug]/page.tsx +++ b/apps/docs/app/guides/troubleshooting/[slug]/page.tsx @@ -4,6 +4,10 @@ import TroubleshootingPage from '~/features/docs/Troubleshooting.page' import { getAllTroubleshootingEntries, getArticleSlug } from '~/features/docs/Troubleshooting.utils' import { PROD_URL } from '~/lib/constants' +// 60 seconds/minute * 60 minutes/hour * 24 hours/day +export const revalidate = 86_400 +export const dynamicParams = false + export default async function TroubleshootingEntryPage({ params: { slug }, }: { diff --git a/apps/docs/app/guides/troubleshooting/page.tsx b/apps/docs/app/guides/troubleshooting/page.tsx index 587e84fa1ae..ac0be072bfe 100644 --- a/apps/docs/app/guides/troubleshooting/page.tsx +++ b/apps/docs/app/guides/troubleshooting/page.tsx @@ -16,6 +16,9 @@ import { TROUBLESHOOTING_CONTAINER_ID } from '~/features/docs/Troubleshooting.ut import { SidebarSkeleton } from '~/layouts/MainSkeleton' import { PROD_URL } from '~/lib/constants' +// 60 seconds/minute * 60 minutes/hour * 24 hours/day +export const revalidate = 86_400 + export default async function GlobalTroubleshootingPage() { const troubleshootingEntries = await getAllTroubleshootingEntries() const keywords = await getAllTroubleshootingKeywords() diff --git a/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx b/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx index 57662e00c61..1d1e733344e 100644 --- a/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx +++ b/apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx @@ -1,15 +1,10 @@ --- title = "How to monitor Postgres and Supavisor connections" -topics = [ -"database", -"supavisor", -] -keywords = [ -"Grafana", -"connections", -"performance", -"pooler", -] +github_url = "https://github.com/orgs/supabase/discussions/27141" +topics = [ "database", "supavisor" ] +keywords = [ "Grafana", "connections", "performance", "pooler" ] +date_created = "2024-06-09" +database_id = "bdd39473-ae02-41f5-b75d-877a79f361f6" --- This guide explains how connections impact your Supabase database's performance and how to optimize them for better resource utilization. diff --git a/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx b/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx index ba1c5dcd9cc..ccb98410f56 100644 --- a/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx +++ b/apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx @@ -1,13 +1,10 @@ --- title = "Interpreting Supabase Grafana Memory Charts" -topics = [ -"database", -] -keywords = [ -"Grafana", -"performance", -"memory" -] +github_url = "https://github.com/orgs/supabase/discussions/27021" +topics = [ "database" ] +keywords = [ "Grafana", "performance", "memory" ] +date_created = "2024-06-05" +database_id = "179d70f3-1e26-4346-9ee8-d340fad382a3" --- > [Supabase Grafana Installation Guide](/docs/guides/platform/metrics#deploying-supabase-grafana) diff --git a/apps/docs/features/docs/Reference.sections.tsx b/apps/docs/features/docs/Reference.sections.tsx index 6bab6c1acc3..79456afd561 100644 --- a/apps/docs/features/docs/Reference.sections.tsx +++ b/apps/docs/features/docs/Reference.sections.tsx @@ -298,7 +298,7 @@ async function ApiEndpointSection({ link, section, servicePath }: ApiEndpointSec
- + {endpointDetails.method} {endpointDetails.path} diff --git a/apps/docs/features/docs/Troubleshooting.page.tsx b/apps/docs/features/docs/Troubleshooting.page.tsx index 5acc242328c..59bfb3d4152 100644 --- a/apps/docs/features/docs/Troubleshooting.page.tsx +++ b/apps/docs/features/docs/Troubleshooting.page.tsx @@ -4,11 +4,15 @@ import Breadcrumbs from '~/components/Breadcrumbs' import { Feedback } from '~/components/Feedback' import { SidebarSkeleton } from '~/layouts/MainSkeleton' import { MDXRemoteBase } from './MdxBase' -import { type ITroubleshootingEntry } from './Troubleshooting.utils' +import { getTroubleshootingUpdatedDates, type ITroubleshootingEntry } from './Troubleshooting.utils' import Link from 'next/link' import { formatError, serializeTroubleshootingSearchParams } from './Troubleshooting.utils.shared' -export default function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { +export default async function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + return (

{entry.data.title}

- {entry.data.updated_at && ( + {dateUpdated && (

- Last edited: {entry.data.updated_at.toLocaleString()} + Last edited: {dateUpdated.toLocaleDateString()}

)}
diff --git a/apps/docs/features/docs/Troubleshooting.script.mjs b/apps/docs/features/docs/Troubleshooting.script.mjs new file mode 100644 index 00000000000..932def8cc23 --- /dev/null +++ b/apps/docs/features/docs/Troubleshooting.script.mjs @@ -0,0 +1,439 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +/** + * Sync new troubleshooting entries from the GitHub repo with GitHub + * Discussions. + */ + +import { createAppAuth } from '@octokit/auth-app' +import { Octokit } from '@octokit/core' +import { createClient } from '@supabase/supabase-js' +import { createHash } from 'crypto' +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { readFile, writeFile } from 'node:fs/promises' +import { stringify } from 'smol-toml' +import toml from 'toml' + +import { + getAllTroubleshootingEntriesInternal as getAllTroubleshootingEntries, + getArticleSlug, +} from './Troubleshooting.utils.common.mjs' + +import 'dotenv/config' + +const REPOSITORY_ID = 'MDEwOlJlcG9zaXRvcnkyMTQ1ODcxOTM=' +const TROUBLESHOOTING_CATEGORY_ID = 'DIC_kwDODMpXOc4CUvEr' +const REPOSITORY_OWNER = 'supabase' +const REPOSITORY_NAME = 'supabase' + +/** + * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingEntry} TroubleshootingEntry + * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingMetadata} TroubleshootingMetadata + */ + +/** + * @type {import('@octokit/core').Octokit} + */ +let octokitInstance +/** + * @type {SupabaseClient} + */ +let supabaseAdminClient + +function octokit() { + if (!octokitInstance) { + octokitInstance = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: process.env.SEARCH_GITHUB_APP_ID, + installationId: process.env.SEARCH_GITHUB_APP_INSTALLATION_ID, + privateKey: process.env.SEARCH_GITHUB_APP_PRIVATE_KEY, + }, + }) + } + + return octokitInstance +} + +export function supabaseAdmin() { + if (!supabaseAdminClient) { + supabaseAdminClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SECRET_KEY + ) + } + + return supabaseAdminClient +} + +async function syncTroubleshootingEntries() { + console.log('[START] Syncing troubleshooting entries to GitHub') + + const [troubleshootingEntries, discussions] = await Promise.all([ + getAllTroubleshootingEntries(), + getAllTroubleshootingDiscussions(), + ]) + + console.log(`[INFO] Found ${troubleshootingEntries.length} existing entries`) + + const tasks = troubleshootingEntries.map(async (entry) => { + const databaseId = entry.data.database_id + if (databaseId.startsWith('pseudo-')) { + // The database entry is faked, so we may need to create a new one. + // There's also an edge case we need to check for: the entry has already + // been created, but the new database ID hasn't been written to the file + // yet. + if (await entryExists(entry)) return + + const discussion = entry.data.github_url + ? await getGithubIdForDiscussion(discussions, entry) + : await createGithubDiscussion(entry) + + let id + try { + id = await insertNewTroubleshootingEntry(entry, discussion) + } catch (error) { + console.error(`[ERROR] Failed to insert new entry for ${entry.data.title}: %O`, error) + console.log( + `[INFO] Rolling back discussion creation for ${entry.data.title} (GitHub ID ${discussion.id})` + ) + await rollbackGithubDiscussion(discussion.id) + throw error + } + + await updateFileId(entry, id) + } else { + // The database entry already exists, so check for updates. + const contentHasChanged = await updateChecksumIfNeeded(entry) + if (contentHasChanged) { + await updateGithubDiscussion(entry) + } + } + }) + + const results = await Promise.allSettled(tasks) + let hasErrors = false + results.forEach((result, index) => { + if (result.status === 'rejected') { + console.error( + `[ERROR] Failed to insert and/or update for ${troubleshootingEntries[index].filePath}:\n%O`, + result.reason + ) + hasErrors = true + } + }) + + return hasErrors +} + +/** + * @param {TroubleshootingEntry} entry + * @throws If the database check fails for some unknown reason + */ +async function entryExists(entry) { + const checksum = calculateChecksum(entry.content) + const { error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('id') + .eq('checksum', checksum) + .single() + + if (error) { + if (error.code === 'PGRST116') { + // No entry found + return false + } + throw error + } + + console.log( + `[INFO] Entry for ${entry.data.title} already exists. Not creating a new one to prevent duplicates.` + ) + return true +} + +/** + * @param {string} content + */ +function calculateChecksum(content) { + // Normalize to ignore changes that don't affect the final displayed content. + const mdast = fromMarkdown(content, { + extensions: [gfm(), mdxjs()], + mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()], + }) + const normalized = toMarkdown(mdast, { extensions: [gfmToMarkdown(), mdxToMarkdown()] }) + + return createHash('sha256').update(normalized).digest('base64') +} + +/** + * @param {TroubleshootingEntry} entry + * + * @param {Object} discussion + * @param {string} discussion.id + * @param {string} discussion.url + * + * @throws If the database insertion fails + */ +async function insertNewTroubleshootingEntry(entry, discussion) { + console.log(`[INFO] Inserting entry for ${entry.data.title} into DB`) + + const now = new Date().toISOString() + const checksum = calculateChecksum(entry.content) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .insert({ + api: entry.data.api, + checksum, + date_created: entry.data.date_created?.toISOString() ?? now, + date_updated: now, + errors: entry.data.errors, + github_id: discussion.id, + github_url: discussion.url, + keywords: entry.data.keywords, + title: entry.data.title, + topics: entry.data.topics, + }) + .select('id') + .single() + if (error) { + throw error + } + + return data.id +} + +/** + * @param {TroubleshootingEntry} entry + * + * @throws If the database update fails + */ +async function updateChecksumIfNeeded(entry) { + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('checksum') + .eq('id', entry.data.database_id) + .single() + if (error) { + throw error + } + + const newChecksum = calculateChecksum(entry.content) + if (data.checksum !== newChecksum) { + console.log(`[INFO] Content changed for ${entry.data.title}. Updating checksum.`) + + const now = new Date().toISOString() + const { error } = await supabaseAdmin() + .from('troubleshooting_entries') + .update({ + checksum: newChecksum, + date_updated: now, + }) + .eq('id', entry.data.database_id) + + if (error) { + throw error + } + + return true + } + + return false +} + +/** + * @param {TroubleshootingEntry} entry + */ +function addCanonicalUrl(entry) { + const docsUrl = 'https://supabase.com/docs/guides/troubleshooting/' + getArticleSlug(entry.data) + const content = + `_This is a copy of a troubleshooting article on Supabase's docs site. It may be missing some details from the original. View the [original article](${docsUrl})._\n\n` + + entry.contentWithoutJsx + return content +} + +/** + * @param {TroubleshootingEntry} entry + */ +async function createGithubDiscussion(entry) { + console.log(`[INFO] Creating GitHub discussion for ${entry.data.title}`) + const content = addCanonicalUrl(entry) + + const mutation = ` + mutation { + createDiscussion(input: { + repositoryId: "${REPOSITORY_ID}", + categoryId: "${TROUBLESHOOTING_CATEGORY_ID}", + body: "${content}", + title: "${entry.data.title}" + }) { + discussion { + id + url + } + } + } + ` + + const { + createDiscussion: { discussion }, + } = await octokit().graphql(mutation) + console.log(`[INFO] Created GitHub discussion for ${entry.data.title}: %s`, discussion.url) + return discussion +} + +/** + * @returns {Promise<{id: string, url: string}[]>} + */ +async function getAllTroubleshootingDiscussions() { + const query = ` + query getDiscussions($cursor: String) { + repository(owner: "${REPOSITORY_OWNER}", name: "${REPOSITORY_NAME}") { + discussions(first: 100, after: $cursor, categoryId: "${TROUBLESHOOTING_CATEGORY_ID}") { + pageInfo { + hasNextPage + endCursor + } + nodes { + id + url + } + } + } + } + ` + + const discussions = [] + let hasNextPage = true + let cursor + + while (hasNextPage) { + const { + repository: { + discussions: { nodes: moreDiscussions, pageInfo }, + }, + } = await octokit().graphql(query, { cursor }) + + discussions.push(...moreDiscussions) + hasNextPage = pageInfo.hasNextPage + cursor = pageInfo.endCursor + } + + return discussions +} + +/** + * @param {{id: string, url: string}[]} discussions + * @param {TroubleshootingEntry} entry + * + * @throws If matching discussion not found + */ +async function getGithubIdForDiscussion(discussions, entry) { + const matchingDiscussion = discussions.find( + (discussion) => discussion.url === entry.data.github_url + ) + if (!matchingDiscussion) { + throw new Error(`No matching discussion found for URL: ${entry.data.github_url}`) + } + return matchingDiscussion +} + +/** + * @param {TroubleshootingEntry} entry + * + * @throws If stored discussion ID for entry not found + */ +async function updateGithubDiscussion(entry) { + console.log(`[INFO] Updating discussion content for ${entry.data.title}`) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('github_id') + .eq('id', entry.data.database_id) + .single() + if (error) { + throw error + } + + const content = addCanonicalUrl(entry) + const mutation = ` + mutation { + updateDiscussion(input: { + discussionId: "${data.github_id}", + body: "${content}", + }) { + discussion { + id + } + } + } + ` + + await octokit().graphql(mutation) + console.log(`[INFO] Updated discussion content for ${entry.data.title}`) +} + +/** @param {string} id */ +async function rollbackGithubDiscussion(id) { + try { + const mutation = ` + mutation { + deleteDiscussion(input: { + id: "${id}", + }) { + discussion { + id + } + } + } + ` + + await octokit().graphql(mutation) + console.log(`[INFO] Rolled back discussion creation for ${id}`) + } catch (error) { + console.error(`[ERROR] Failed to rollback discussion creation for ${id}: %O`, error) + } +} + +/** + * @param {TroubleshootingEntry} entry + * @param {string} id + * + * @throws Passes through readFile and writeFile errors without catching + */ +async function updateFileId(entry, id) { + console.log(`[INFO] Writing database ID to file for ${entry.filePath}`) + + const fileContents = await readFile(entry.filePath, 'utf-8') + const { data, content } = matter(fileContents, { + language: 'toml', + engines: { toml: toml.parse.bind(toml) }, + }) + data.database_id = id + + const newFrontmatter = stringify(data) + const newContent = `---\n${newFrontmatter}\n---\n${content}` + + await writeFile(entry.filePath, newContent) +} + +async function main() { + try { + const hasErrors = await syncTroubleshootingEntries() + if (hasErrors) { + process.exit(1) + } + } catch (error) { + console.error(`[ERROR] %O`, error) + process.exit(1) + } +} + +main() diff --git a/apps/docs/features/docs/Troubleshooting.ui.tsx b/apps/docs/features/docs/Troubleshooting.ui.tsx index 06ca618cff2..b3fec96bff8 100644 --- a/apps/docs/features/docs/Troubleshooting.ui.tsx +++ b/apps/docs/features/docs/Troubleshooting.ui.tsx @@ -2,12 +2,20 @@ import { Wrench } from 'lucide-react' import Link from 'next/link' import { type PropsWithChildren, useCallback } from 'react' -import { type ITroubleshootingEntry, getArticleSlug } from './Troubleshooting.utils' +import { + type ITroubleshootingEntry, + getArticleSlug, + getTroubleshootingUpdatedDates, +} from './Troubleshooting.utils' import { TroubleshootingFilter } from './Troubleshooting.ui.client' import { formatError, TROUBLESHOOTING_DATA_ATTRIBUTES } from './Troubleshooting.utils.shared' import { cn } from 'ui' -export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { +export async function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + const keywords = [...entry.data.topics, ...(entry.data.keywords ?? [])] const attributes = { [TROUBLESHOOTING_DATA_ATTRIBUTES.QUERY_ATTRIBUTE]: @@ -58,14 +66,13 @@ export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry ))}
- {entry.data.updated_at && + {dateUpdated && (() => { - const date = new Date(entry.data.updated_at) const options = { month: 'short', day: 'numeric' } as Intl.DateTimeFormatOptions - if (date.getFullYear() !== new Date().getFullYear()) { + if (dateUpdated.getFullYear() !== new Date().getFullYear()) { options.year = 'numeric' } - return date.toLocaleDateString(undefined, options) + return dateUpdated.toLocaleDateString(undefined, options) })()}
diff --git a/apps/docs/features/docs/Troubleshooting.utils.common.mjs b/apps/docs/features/docs/Troubleshooting.utils.common.mjs new file mode 100644 index 00000000000..e2571f07819 --- /dev/null +++ b/apps/docs/features/docs/Troubleshooting.utils.common.mjs @@ -0,0 +1,185 @@ +/** + * This file is for utils needed in both the Next.js app build and the + * troubleshooting sync script. Because of unsolved problems with imports, the + * script is a mjs file instead of a ts file. Any dependencies that are needed + * in both places are defined here, and then typed in Troubleshooting.utils.ts + * as required. + */ + +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { readdir, readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import toml from 'toml' +import { visit } from 'unist-util-visit' +import { v4 as uuidv4 } from 'uuid' +import { z } from 'zod' + +export const TROUBLESHOOTING_DIRECTORY = join(process.cwd(), 'content/troubleshooting') + +/** + * @typedef {Object} TroubleshootingEntry + * @property {string} filePath + * @property {string} content + * @property {string} contentWithoutJsx + * @property {TroubleshootingMetadata} data + */ + +/** + * @typedef {Object} TroubleshootingMetadata + * @property {string} title + * @property {string[]} topics + * @property {string[]} [keywords] + * @property {Object} [api] + * @property {string[]} [api.sdk] + * @property {string[]} [api.management_api] + * @property {string[]} [api.cli] + * @property {AssociatedError[]} [errors] + * @property {string} database_id + * @property {string} [github_url] + * @property {Date} [date_created] + */ + +/** + * @typedef {Object} AssociatedError + * @property {number} [http_status_code] + * @property {string} [code] + * @property {string} [message] + */ + +export const TroubleshootingSchema = z + .object({ + title: z.string(), + topics: z.array( + z.enum([ + 'ai', + 'auth', + 'branching', + 'cli', + 'database', + 'functions', + 'platform', + 'realtime', + 'self-hosting', + 'storage', + 'studio', + 'supavisor', + 'terraform', + ]) + ), + keywords: z.array(z.string()).optional(), + api: z + .object({ + sdk: z.array(z.string()).optional(), + management_api: z.array(z.string()).optional(), + cli: z.array(z.string()).optional(), + }) + .strict() + .optional(), + errors: z + .array( + z + .object({ + http_status_code: z.number().optional(), + code: z.string().optional(), + message: z.string().optional(), + }) + .strict() + ) + .optional(), + database_id: z.string().default(`pseudo-${uuidv4()}`), + github_url: z.string().url().optional(), + date_created: z.date({ coerce: true }).optional(), + }) + .strict() + +/* + * @param {unknown} troubleshootingMetadata + */ +function validateTroubleshootingMetadata(troubleshootingMetadata) { + return TroubleshootingSchema.safeParse(troubleshootingMetadata) +} + +/* + * @returns {Promise} + */ +export async function getAllTroubleshootingEntriesInternal() { + const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, { + recursive: true, + }) + const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => { + const isHidden = entry.startsWith('_') + if (isHidden) return null + + const filePath = join(TROUBLESHOOTING_DIRECTORY, entry) + + const isFile = (await stat(filePath)).isFile() + if (!isFile) return null + + const fileContents = await readFile(filePath, 'utf-8') + const { content, data: frontmatter } = matter(fileContents, { + language: 'toml', + engines: { toml: toml.parse.bind(toml) }, + }) + + const parseResult = validateTroubleshootingMetadata(frontmatter) + if ('error' in parseResult) { + console.error( + `Error validating troubleshooting metadata\nEntry:%O\nError:%O`, + frontmatter, + parseResult.error + ) + return null + } + + const mdxTree = fromMarkdown(content, { + extensions: [gfm(), mdxjs()], + mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()], + }) + visit(mdxTree, (node) => { + if ('children' in node) { + node.children = node.children.filter( + (child) => + ![ + 'mdxJsxFlowExpression', + 'mdxJsxTextExpression', + 'mdxFlowExpression', + 'mdxTextExpression', + 'mdxJsxFlowElement', + 'mdxJsxTextElement', + 'mdxJsxExpressionAttribute', + 'mdxJsxAttribute', + 'mdxJsxAttributeValueExpression', + 'mdxjsEsm', + ].includes(child.type) + ) + } + }) + const contentWithoutJsx = toMarkdown(mdxTree, { + extensions: [gfmToMarkdown()], + }) + + return { + filePath, + content, + contentWithoutJsx, + data: parseResult.data, + } + }) + + return (await Promise.all(troubleshootingFiles)).filter(Boolean) +} + +/** + * @param {TroubleshootingMetadata} meta + */ +export function getArticleSlug(meta) { + const slugifiedTitle = meta.title.toLowerCase().replace(/\s+/g, '-') + const escapedTitle = encodeURIComponent(slugifiedTitle) + return escapedTitle +} diff --git a/apps/docs/features/docs/Troubleshooting.utils.ts b/apps/docs/features/docs/Troubleshooting.utils.ts index 0b2c755303a..2f858fcd9dd 100644 --- a/apps/docs/features/docs/Troubleshooting.utils.ts +++ b/apps/docs/features/docs/Troubleshooting.utils.ts @@ -1,137 +1,39 @@ -import matter from 'gray-matter' -import { fromMarkdown } from 'mdast-util-from-markdown' -import { mdxFromMarkdown } from 'mdast-util-mdx' -import { toMarkdown } from 'mdast-util-to-markdown' -import { mdxjs } from 'micromark-extension-mdxjs' -import { readdir, readFile, stat } from 'node:fs/promises' -import { join } from 'node:path' -import toml from 'toml' -import { visit } from 'unist-util-visit' -import { v4 as uuidv4 } from 'uuid' +import { cache } from 'react' import { z } from 'zod' -import { DOCS_DIRECTORY } from 'lib/docs' import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs' +import { supabaseAdmin } from '~/lib/supabaseAdmin' +import { + getAllTroubleshootingEntriesInternal, + getArticleSlug as getArticleSlugInternal, + TroubleshootingSchema, + TROUBLESHOOTING_DIRECTORY, +} from './Troubleshooting.utils.common.mjs' import { formatError } from './Troubleshooting.utils.shared' -const TROUBLESHOOTING_DIRECTORY = join(DOCS_DIRECTORY, 'content/troubleshooting') +// We don't have strictNullChecks on because we still have too many violations +// in the code base, so required types aren't typed properly by Zod. +export type ITroubleshootingMetadata = z.infer & + Pick>, 'title' | 'topics' | 'database_id'> -const TroubleshootingSchema = z - .object({ - title: z.string(), - topics: z.array( - z.enum([ - 'ai', - 'auth', - 'branching', - 'cli', - 'database', - 'functions', - 'platform', - 'realtime', - 'self-hosting', - 'storage', - 'studio', - 'supavisor', - 'terraform', - ]) - ), - keywords: z.array(z.string()).optional(), - api: z - .object({ - sdk: z.array(z.string()).optional(), - management_api: z.array(z.string()).optional(), - cli: z.array(z.string()).optional(), - }) - .strict() - .optional(), - errors: z - .array( - z - .object({ - http_status_code: z.number().optional(), - code: z.string().optional(), - message: z.string().optional(), - }) - .strict() - ) - .optional(), - database_id: z.string().uuid().default(uuidv4()), - created_at: z.date({ coerce: true }).optional(), - updated_at: z.date({ coerce: true }).optional(), - }) - .strict() - -export type ITroubleshootingMetadata = z.infer - -const validateTroubleshootingMetadata = (troubleshootingMetadata: unknown) => { - return TroubleshootingSchema.safeParse(troubleshootingMetadata) +export interface ITroubleshootingEntry { + filePath: string + content: string + contentWithoutJsx: string + data: ITroubleshootingMetadata } -async function getAllTroubleshootingEntriesInternal() { - const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, { - recursive: true, - }) - const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => { - const isHidden = entry.startsWith('_') - if (isHidden) return null +export const getArticleSlug = getArticleSlugInternal - const isFile = (await stat(join(TROUBLESHOOTING_DIRECTORY, entry))).isFile() - if (!isFile) return null - - const fileContents = await readFile(join(TROUBLESHOOTING_DIRECTORY, entry), 'utf-8') - const { content, data: frontmatter } = matter(fileContents, { - language: 'toml', - engines: { toml: toml.parse.bind(toml) }, - }) - - const parseResult = validateTroubleshootingMetadata(frontmatter) - if ('error' in parseResult) { - console.error(`Error validating troubleshooting metadata for ${entry}:`, parseResult.error) - return null - } - - const mdxTree = fromMarkdown(content, { - extensions: [mdxjs()], - mdastExtensions: [mdxFromMarkdown()], - }) - visit(mdxTree, (node) => { - if ('children' in node) { - node.children = node.children.filter( - (child) => - ![ - 'mdxJsxFlowExpression', - 'mdxJsxTextExpression', - 'mdxFlowExpression', - 'mdxTextExpression', - 'mdxJsxFlowElement', - 'mdxJsxTextElement', - 'mdxJsxExpressionAttribute', - 'mdxJsxAttribute', - 'mdxJsxAttributeValueExpression', - 'mdxjsEsm', - ].includes(child.type) - ) - } - }) - const contentWithoutJsx = toMarkdown(mdxTree) - - return { - content, - contentWithoutJsx, - data: parseResult.data, - } - }) - - return (await Promise.all(troubleshootingFiles)).filter(Boolean) +async function getAllTroubleshootingEntriesTyped() { + const result = (await getAllTroubleshootingEntriesInternal()) as ITroubleshootingEntry[] + return result } - export const getAllTroubleshootingEntries = cache_fullProcess_withDevCacheBust( - getAllTroubleshootingEntriesInternal, + getAllTroubleshootingEntriesTyped, TROUBLESHOOTING_DIRECTORY, () => JSON.stringify([]) ) -export type ITroubleshootingEntry = Awaited>[number] export async function getAllTroubleshootingKeywords() { const entries = await getAllTroubleshootingEntries() @@ -185,8 +87,22 @@ export async function getAllTroubleshootingErrors() { return Array.from(allErrors).sort(sortErrors) } -export function getArticleSlug(entry: ITroubleshootingMetadata) { - const slugifiedTitle = entry.title.toLowerCase().replace(/\s+/g, '-') - const escapedTitle = encodeURIComponent(slugifiedTitle) - return escapedTitle +async function getTroubleshootingUpdatedDatesInternal() { + const databaseIds = (await getAllTroubleshootingEntries()) + .map((entry) => entry.data.database_id) + .filter((id) => !id.startsWith('pseudo-')) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('id, date_updated') + .in('id', databaseIds) + if (error) { + console.error(error) + } + + return (data ?? []).reduce((acc, entry) => { + acc.set(entry.id, new Date(entry.date_updated)) + return acc + }, new Map()) } +export const getTroubleshootingUpdatedDates = cache(getTroubleshootingUpdatedDatesInternal) diff --git a/apps/docs/lib/supabaseAdmin.ts b/apps/docs/lib/supabaseAdmin.ts new file mode 100644 index 00000000000..2c58ea65395 --- /dev/null +++ b/apps/docs/lib/supabaseAdmin.ts @@ -0,0 +1,18 @@ +import 'server-only' + +import { createClient, type SupabaseClient } from '@supabase/supabase-js' + +import { type Database } from 'common' + +let supabaseAdminClient: SupabaseClient | null = null + +export function supabaseAdmin() { + if (!supabaseAdminClient) { + supabaseAdminClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SECRET_KEY + ) + } + + return supabaseAdminClient +} diff --git a/apps/docs/package.json b/apps/docs/package.json index 62f16690760..9cf97308475 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -18,6 +18,7 @@ "test:smoke": "npm run codegen:references && vitest -t \"prod smoke test\"", "embeddings": "tsx scripts/search/generate-embeddings.ts", "embeddings:refresh": "npm run embeddings -- --refresh", + "troubleshooting:sync": "node features/docs/Troubleshooting.script.mjs", "last-changed": "tsx scripts/last-changed.ts", "last-changed:reset": "npm run last-changed -- --reset", "codegen:references": "tsx features/docs/Reference.generated.script.ts", @@ -127,6 +128,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", diff --git a/package-lock.json b/package-lock.json index 2a1db90666b..2772cec9bae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -958,6 +958,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", @@ -39644,6 +39645,18 @@ "npm": ">= 3.0.0" } }, + "node_modules/smol-toml": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.0.tgz", + "integrity": "sha512-tWpi2TsODPScmi48b/OQZGi2lgUmBCHy6SZrhi/FdnnHiU1GwebbCfuQuxsC3nHaLwtYeJGPrDZDIeodDOc4pA==", + "dev": true, + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, "node_modules/snake-case": { "version": "3.0.4", "dev": true, diff --git a/packages/common/database-types.ts b/packages/common/database-types.ts index c13e9a2de83..79ee7683f14 100644 --- a/packages/common/database-types.ts +++ b/packages/common/database-types.ts @@ -318,9 +318,11 @@ export type Database = { troubleshooting_entries: { Row: { api: Json | null + checksum: string date_created: string date_updated: string errors: Json[] | null + github_id: string github_url: string id: string keywords: string[] | null @@ -329,9 +331,11 @@ export type Database = { } Insert: { api?: Json | null + checksum: string date_created?: string date_updated?: string errors?: Json[] | null + github_id: string github_url: string id?: string keywords?: string[] | null @@ -340,9 +344,11 @@ export type Database = { } Update: { api?: Json | null + checksum?: string date_created?: string date_updated?: string errors?: Json[] | null + github_id?: string github_url?: string id?: string keywords?: string[] | null @@ -464,6 +470,33 @@ export type Database = { } Returns: unknown } + json_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonb_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonschema_is_valid: { + Args: { + schema: Json + } + Returns: boolean + } + jsonschema_validation_errors: { + Args: { + schema: Json + instance: Json + } + Returns: string[] + } match_page_sections_v2: { Args: { embedding: string @@ -491,6 +524,12 @@ export type Database = { } Returns: string } + validate_troubleshooting_errors: { + Args: { + errors: Json[] + } + Returns: boolean + } vector_avg: { Args: { "": number[]