feat(docs): sync troubleshooting

When a troubleshooting entry is created or updated in this repo, sync it to GitHub. This creates a new troubleshooting discussion if one does not exist, and updates the content on an existing troubleshooting discussion if it has changed.

To preserve the connections between GitHub discussions and Markdown files, record the discussion ID and URL to the database. The database ID is then written to the Markdown file, and a new PR is generated to commit the new Markdown with ID.

There is currently no mechanism to delete outdated discussions. That will be a follow-up.

.github/workflows/docs-sync-troubleshooting.yml

@@ -0,0 +1,54 @@
+name: '[Docs] Sync troubleshooting guides to GitHub Discussions'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'apps/docs/content/troubleshooting/**'
+  workflow_dispatch:
+  # testing only, delete later
+  pull_request:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-troubleshooting:
+    runs-on: ubuntu-latest
+
+    env:
+      SEARCH_GITHUB_APP_ID: ${{ secrets.SEARCH_GITHUB_APP_ID }}
+      SEARCH_GITHUB_APP_INSTALLATION_ID: ${{ secrets.SEARCH_GITHUB_APP_INSTALLATION_ID }}
+      SEARCH_GITHUB_APP_PRIVATE_KEY: ${{ secrets.SEARCH_GITHUB_APP_PRIVATE_KEY }}
+      NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}
+      SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            apps/docs
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run Troubleshooting script
+        run: npm --prefix=apps/docs run troubleshooting:sync
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
+        with:
+          # for testing on pull_request only
+          base: ${{ github.head_ref }}
+          commit-message: '[bot] sync troubleshooting guides to db'
+          title: '[bot] sync troubleshooting guides to db'
+          author: 'github-docs-sync-bot <github-docs-sync-bot@supabase.com>'
+          branch: 'bot/docs-sync-troubleshooting'

apps/docs/content/troubleshooting/monitor-supavisor-postgres-connections.mdx

@@ -1,5 +1,6 @@
 ---
 title = "How to monitor Postgres and Supavisor connections"
+github_url = "https://github.com/orgs/supabase/discussions/27141"
 topics = [
 "database",
 "supavisor",
@@ -10,6 +11,7 @@ keywords = [
 "performance",
 "pooler",
 ]
+date_created = "2024-06-09"
 ---
 
 This guide explains how connections impact your Supabase database's performance and how to optimize them for better resource utilization.

apps/docs/content/troubleshooting/supabase-grafana-memory-charts.mdx

@@ -1,5 +1,6 @@
 ---
 title = "Interpreting Supabase Grafana Memory Charts"
+github_url = "https://github.com/orgs/supabase/discussions/27021"
 topics = [
 "database",
 ]
@@ -8,6 +9,7 @@ keywords = [
 "performance",
 "memory"
 ]
+date_created = "2024-06-05"
 ---
 
 > [Supabase Grafana Installation Guide](/docs/guides/platform/metrics#deploying-supabase-grafana)

apps/docs/features/docs/Reference.sections.tsx

@@ -298,7 +298,7 @@ async function ApiEndpointSection({ link, section, servicePath }: ApiEndpointSec
       <StickyHeader title={endpointDetails.summary} className="col-[1_/_-1]" />
       <div className="flex flex-col gap-12">
         <div className="flex items-center gap-2">
-          <span className="font-mono uppercase text-sm whitespace-nowrap bg-foreground text-background rounded-full font-mono font-medium px-2 py-0.5">
+          <span className="uppercase text-sm whitespace-nowrap bg-foreground text-background rounded-full font-mono font-medium px-2 py-0.5">
             {endpointDetails.method}
           </span>
           <code className="text-foreground-lighter break-all">{endpointDetails.path}</code>

apps/docs/features/docs/Troubleshooting.page.tsx

@@ -4,11 +4,15 @@ import Breadcrumbs from '~/components/Breadcrumbs'
 import { Feedback } from '~/components/Feedback'
 import { SidebarSkeleton } from '~/layouts/MainSkeleton'
 import { MDXRemoteBase } from './MdxBase'
-import { type ITroubleshootingEntry } from './Troubleshooting.utils'
+import { getTroubleshootingUpdatedDates, type ITroubleshootingEntry } from './Troubleshooting.utils'
 import Link from 'next/link'
 import { formatError, serializeTroubleshootingSearchParams } from './Troubleshooting.utils.shared'
 
-export default function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) {
+export default async function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) {
+  const dateUpdated = entry.data.database_id.startsWith('pseudo-')
+    ? new Date()
+    : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id)
+
   return (
     <SidebarSkeleton
       hideSideNav
@@ -18,9 +22,9 @@ export default function TroubleshootingPage({ entry }: { entry: ITroubleshooting
         <Breadcrumbs minLength={1} forceDisplayOnMobile />
         <article className="prose max-w-none mt-4">
           <h1>{entry.data.title}</h1>
-          {entry.data.updated_at && (
+          {dateUpdated && (
             <p className="text-sm text-foreground-lighter">
-              Last edited: {entry.data.updated_at.toLocaleString()}
+              Last edited: {dateUpdated.toLocaleDateString()}
             </p>
           )}
           <hr className="my-7" aria-hidden />

apps/docs/features/docs/Troubleshooting.script.mjs

@@ -0,0 +1,392 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+
+/**
+ * Sync new troubleshooting entries from the GitHub repo with GitHub
+ * Discussions.
+ */
+
+import { createAppAuth } from '@octokit/auth-app'
+import { Octokit } from '@octokit/core'
+import { createClient } from '@supabase/supabase-js'
+import { createHash } from 'crypto'
+import matter from 'gray-matter'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { gfm } from 'micromark-extension-gfm'
+import { mdxjs } from 'micromark-extension-mdxjs'
+import { readFile, writeFile } from 'node:fs/promises'
+import { stringify } from 'smol-toml'
+import toml from 'toml'
+
+import {
+  getAllTroubleshootingEntriesInternal as getAllTroubleshootingEntries,
+  getArticleSlug,
+} from './Troubleshooting.utils.common.mjs'
+
+import 'dotenv/config'
+
+/**
+ * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingEntry} TroubleshootingEntry
+ * @typedef {import('./Troubleshooting.utils.common.mjs').TroubleshootingMetadata} TroubleshootingMetadata
+ */
+
+/**
+ * @type {import('@octokit/core').Octokit}
+ */
+let octokitInstance
+/**
+ * @type {SupabaseClient<import('../../../../packages/common').Database>}
+ */
+let supabaseAdminClient
+
+function octokit() {
+  if (!octokitInstance) {
+    octokitInstance = new Octokit({
+      authStrategy: createAppAuth,
+      auth: {
+        appId: process.env.SEARCH_GITHUB_APP_ID,
+        installationId: process.env.SEARCH_GITHUB_APP_INSTALLATION_ID,
+        privateKey: process.env.SEARCH_GITHUB_APP_PRIVATE_KEY,
+      },
+    })
+  }
+
+  return octokitInstance
+}
+
+export function supabaseAdmin() {
+  if (!supabaseAdminClient) {
+    supabaseAdminClient = createClient(
+      process.env.NEXT_PUBLIC_SUPABASE_URL,
+      process.env.SUPABASE_SECRET_KEY
+    )
+  }
+
+  return supabaseAdminClient
+}
+
+async function syncTroubleshootingEntries() {
+  console.log('[START] Syncing troubleshooting entries to GitHub')
+
+  const [troubleshootingEntries, discussions] = await Promise.all([
+    getAllTroubleshootingEntries(),
+    getAllTroubleshootingDiscussions(),
+  ])
+
+  console.log(`[INFO] Found ${troubleshootingEntries.length} existing entries`)
+
+  const tasks = troubleshootingEntries.map(async (entry) => {
+    const databaseId = entry.data.database_id
+    if (databaseId.startsWith('pseudo-')) {
+      // The database entry is faked, so we may need to create a new one.
+      // There's also an edge case we need to check for: the entry has already
+      // been created, but the new database ID hasn't been written to the file
+      // yet.
+      if (await entryExists(entry)) return
+
+      const discussion = entry.data.github_url
+        ? await getGithubIdForDiscussion(discussions, entry)
+        : await createGithubDiscussion(entry)
+      const id = await insertNewTroubleshootingEntry(entry, discussion)
+      await updateFileId(entry, id)
+    } else {
+      // The database entry already exists, so check for updates.
+      const contentHasChanged = await updateChecksumIfNeeded(entry)
+      if (contentHasChanged) {
+        await updateGithubDiscussion(entry)
+      }
+    }
+  })
+
+  const results = await Promise.allSettled(tasks)
+  let hasErrors = false
+  results.forEach((result, index) => {
+    if (result.status === 'rejected') {
+      console.error(
+        `Failed to insert and/or update GitHub discussion for ${troubleshootingEntries[index].filePath}:\n%O`,
+        result.reason
+      )
+      hasErrors = true
+    }
+  })
+
+  return hasErrors
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ * @throws If the database check fails for some unknown reason
+ */
+async function entryExists(entry) {
+  const checksum = calculateChecksum(entry.content)
+  const { error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('id')
+    .eq('checksum', checksum)
+    .single()
+
+  if (error) {
+    if (error.code === 'PGRST116') {
+      // No entry found
+      return false
+    }
+    throw error
+  }
+
+  console.log(
+    `Entry for ${entry.data.title} already exists. Not creating a new one to prevent duplicates.`
+  )
+  return true
+}
+
+/**
+ * @param {string} content
+ */
+function calculateChecksum(content) {
+  // Normalize to ignore changes that don't affect the final displayed content.
+  const mdast = fromMarkdown(content, {
+    extensions: [gfm(), mdxjs()],
+    mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()],
+  })
+  const normalized = toMarkdown(mdast, { extensions: [gfmToMarkdown(), mdxToMarkdown()] })
+
+  return createHash('sha256').update(normalized).digest('base64')
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ *
+ * @param {Object} discussion
+ * @param {string} discussion.id
+ * @param {string} discussion.url
+ *
+ * @throws If the database insertion fails
+ */
+async function insertNewTroubleshootingEntry(entry, discussion) {
+  const now = new Date().toISOString()
+  const checksum = calculateChecksum(entry.content)
+
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .insert({
+      api: entry.data.api,
+      checksum,
+      date_created: entry.data.date_created?.toISOString() ?? now,
+      date_updated: now,
+      errors: entry.data.errors,
+      github_id: discussion.id,
+      github_url: discussion.url,
+      keywords: entry.data.keywords,
+      title: entry.data.title,
+      topics: entry.data.topics,
+    })
+    .select('id')
+    .single()
+  if (error) {
+    throw error
+  }
+
+  return data.id
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ *
+ * @throws If the database update fails
+ */
+async function updateChecksumIfNeeded(entry) {
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('checksum')
+    .eq('id', entry.data.database_id)
+    .single()
+  if (error) {
+    throw error
+  }
+
+  const newChecksum = calculateChecksum(entry.content)
+  if (data.checksum !== newChecksum) {
+    const now = new Date().toISOString()
+    const { error } = await supabaseAdmin()
+      .from('troubleshooting_entries')
+      .update({
+        checksum: newChecksum,
+        date_updated: now,
+      })
+      .eq('id', entry.data.database_id)
+
+    if (error) {
+      throw error
+    }
+
+    return true
+  }
+
+  return false
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ */
+function addCanonicalUrl(entry) {
+  const docsUrl = 'https://supabase.com/docs/guides/troubleshooting/' + getArticleSlug(entry.data)
+  const content =
+    entry.contentWithoutJsx +
+    `\n\n_This is a copy of a troubleshooting article on Supabase's docs site. It may be missing some details from the original. View the [original article](${docsUrl})._`
+  return content
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ */
+async function createGithubDiscussion(entry) {
+  console.log(`Creating GitHub discussion for ${entry.data.title}`)
+  const content = addCanonicalUrl(entry)
+
+  const mutation = `
+    mutation {
+      createDiscussion(input: {
+        repositoryId: "MDEwOlJlcG9zaXRvcnkyMTQ1ODcxOTM=",
+        categoryId: "DIC_kwDODMpXOc4CUvEr",
+        body: "${content}",
+        title: "${entry.data.title}"
+      }) {
+        discussion {
+          id
+          url
+        }
+      }
+    }
+    `
+
+  const { discussion } = await octokit().graphql(mutation)
+  console.log(`Created GitHub discussion for ${entry.data.title}`)
+  return discussion
+}
+
+/**
+ * @returns {Promise<{id: string, url: string}[]>}
+ */
+async function getAllTroubleshootingDiscussions() {
+  const troubleshootingCategoryId = 'DIC_kwDODMpXOc4CUvEr'
+  const query = `
+    query getDiscussions($cursor: String) {
+      repository(owner: "supabase", name: "supabase") {
+        discussions(first: 100, after: $cursor, categoryId: "${troubleshootingCategoryId}") {
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+          nodes {
+            id
+            url
+          }
+        }
+      }
+    }
+  `
+
+  const discussions = []
+  let hasNextPage = true
+  let cursor
+
+  while (hasNextPage) {
+    const {
+      repository: {
+        discussions: { nodes: moreDiscussions, pageInfo },
+      },
+    } = await octokit().graphql(query, { cursor })
+
+    discussions.push(...moreDiscussions)
+    hasNextPage = pageInfo.hasNextPage
+    cursor = pageInfo.endCursor
+  }
+
+  return discussions
+}
+
+/**
+ * @param {{id: string, url: string}[]} discussions
+ * @param {TroubleshootingEntry} entry
+ *
+ * @throws If matching discussion not found
+ */
+async function getGithubIdForDiscussion(discussions, entry) {
+  const matchingDiscussion = discussions.find(
+    (discussion) => discussion.url === entry.data.github_url
+  )
+  if (!matchingDiscussion) {
+    throw new Error(`No matching discussion found for URL: ${entry.data.github_url}`)
+  }
+  return matchingDiscussion
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ *
+ * @throws If stored discussion ID for entry not found
+ */
+async function updateGithubDiscussion(entry) {
+  console.log(`Updating discussion content for ${entry.data.title}`)
+
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('github_id')
+    .eq('id', entry.data.database_id)
+    .single()
+  if (error) {
+    throw error
+  }
+
+  const content = addCanonicalUrl(entry)
+  const mutation = `
+    mutation {
+      updateDiscussion(input: {
+        discussionId: "${data.github_id}",
+        body: "${content}",
+      }) {
+      }
+    }
+    `
+
+  await octokit().graphql(mutation)
+  console.log(`Updated discussion content for ${entry.data.title}`)
+}
+
+/**
+ * @param {TroubleshootingEntry} entry
+ * @param {string} id
+ *
+ * @throws Passes through readFile and writeFile errors without catching
+ */
+async function updateFileId(entry, id) {
+  console.log(`Writing database ID to file for ${entry.filePath}`)
+
+  const fileContents = await readFile(entry.filePath, 'utf-8')
+  const { data, content } = matter(fileContents, {
+    language: 'toml',
+    engines: { toml: toml.parse.bind(toml) },
+  })
+  data.database_id = id
+
+  const newFrontmatter = stringify(data)
+  const newContent = `---\n${newFrontmatter}\n---\n${content}`
+
+  await writeFile(entry.filePath, newContent)
+}
+
+async function main() {
+  try {
+    const hasErrors = await syncTroubleshootingEntries()
+    if (hasErrors) {
+      process.exit(1)
+    }
+  } catch (error) {
+    console.error(error)
+    process.exit(1)
+  }
+}
+
+main()

apps/docs/features/docs/Troubleshooting.ui.tsx

@@ -2,12 +2,20 @@ import { Wrench } from 'lucide-react'
 import Link from 'next/link'
 import { type PropsWithChildren, useCallback } from 'react'
 
-import { type ITroubleshootingEntry, getArticleSlug } from './Troubleshooting.utils'
+import {
+  type ITroubleshootingEntry,
+  getArticleSlug,
+  getTroubleshootingUpdatedDates,
+} from './Troubleshooting.utils'
 import { TroubleshootingFilter } from './Troubleshooting.ui.client'
 import { formatError, TROUBLESHOOTING_DATA_ATTRIBUTES } from './Troubleshooting.utils.shared'
 import { cn } from 'ui'
 
-export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) {
+export async function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) {
+  const dateUpdated = entry.data.database_id.startsWith('pseudo-')
+    ? new Date()
+    : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id)
+
   const keywords = [...entry.data.topics, ...(entry.data.keywords ?? [])]
   const attributes = {
     [TROUBLESHOOTING_DATA_ATTRIBUTES.QUERY_ATTRIBUTE]:
@@ -58,14 +66,13 @@ export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry
             ))}
           </div>
           <div className="basis-l8 flex-shrink-0 flex-grow-0 truncate text-sm text-foreground-lighter">
-            {entry.data.updated_at &&
+            {dateUpdated &&
               (() => {
-                const date = new Date(entry.data.updated_at)
                 const options = { month: 'short', day: 'numeric' } as Intl.DateTimeFormatOptions
-                if (date.getFullYear() !== new Date().getFullYear()) {
+                if (dateUpdated.getFullYear() !== new Date().getFullYear()) {
                   options.year = 'numeric'
                 }
-                return date.toLocaleDateString(undefined, options)
+                return dateUpdated.toLocaleDateString(undefined, options)
               })()}
           </div>
         </div>

apps/docs/features/docs/Troubleshooting.utils.common.mjs

@@ -0,0 +1,185 @@
+/**
+ * This file is for utils needed in both the Next.js app build and the
+ * troubleshooting sync script. Because of unsolved problems with imports, the
+ * script is a mjs file instead of a ts file. Any dependencies that are needed
+ * in both places are defined here, and then typed in Troubleshooting.utils.ts
+ * as required.
+ */
+
+import matter from 'gray-matter'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { mdxFromMarkdown } from 'mdast-util-mdx'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { gfm } from 'micromark-extension-gfm'
+import { mdxjs } from 'micromark-extension-mdxjs'
+import { readdir, readFile, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import toml from 'toml'
+import { visit } from 'unist-util-visit'
+import { v4 as uuidv4 } from 'uuid'
+import { z } from 'zod'
+
+export const TROUBLESHOOTING_DIRECTORY = join(process.cwd(), 'content/troubleshooting')
+
+/**
+ * @typedef {Object} TroubleshootingEntry
+ * @property {string} filePath
+ * @property {string} content
+ * @property {string} contentWithoutJsx
+ * @property {TroubleshootingMetadata} data
+ */
+
+/**
+ * @typedef {Object} TroubleshootingMetadata
+ * @property {string} title
+ * @property {string[]} topics
+ * @property {string[]} [keywords]
+ * @property {Object} [api]
+ * @property {string[]} [api.sdk]
+ * @property {string[]} [api.management_api]
+ * @property {string[]} [api.cli]
+ * @property {AssociatedError[]} [errors]
+ * @property {string} database_id
+ * @property {string} [github_url]
+ * @property {Date} [date_created]
+ */
+
+/**
+ * @typedef {Object} AssociatedError
+ * @property {number} [http_status_code]
+ * @property {string} [code]
+ * @property {string} [message]
+ */
+
+export const TroubleshootingSchema = z
+  .object({
+    title: z.string(),
+    topics: z.array(
+      z.enum([
+        'ai',
+        'auth',
+        'branching',
+        'cli',
+        'database',
+        'functions',
+        'platform',
+        'realtime',
+        'self-hosting',
+        'storage',
+        'studio',
+        'supavisor',
+        'terraform',
+      ])
+    ),
+    keywords: z.array(z.string()).optional(),
+    api: z
+      .object({
+        sdk: z.array(z.string()).optional(),
+        management_api: z.array(z.string()).optional(),
+        cli: z.array(z.string()).optional(),
+      })
+      .strict()
+      .optional(),
+    errors: z
+      .array(
+        z
+          .object({
+            http_status_code: z.number().optional(),
+            code: z.string().optional(),
+            message: z.string().optional(),
+          })
+          .strict()
+      )
+      .optional(),
+    database_id: z.string().default(`pseudo-${uuidv4()}`),
+    github_url: z.string().url().optional(),
+    date_created: z.date({ coerce: true }).optional(),
+  })
+  .strict()
+
+/*
+ * @param {unknown} troubleshootingMetadata
+ */
+function validateTroubleshootingMetadata(troubleshootingMetadata) {
+  return TroubleshootingSchema.safeParse(troubleshootingMetadata)
+}
+
+/*
+ * @returns {Promise<TroubleshootingEntry[]>}
+ */
+export async function getAllTroubleshootingEntriesInternal() {
+  const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, {
+    recursive: true,
+  })
+  const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => {
+    const isHidden = entry.startsWith('_')
+    if (isHidden) return null
+
+    const filePath = join(TROUBLESHOOTING_DIRECTORY, entry)
+
+    const isFile = (await stat(filePath)).isFile()
+    if (!isFile) return null
+
+    const fileContents = await readFile(filePath, 'utf-8')
+    const { content, data: frontmatter } = matter(fileContents, {
+      language: 'toml',
+      engines: { toml: toml.parse.bind(toml) },
+    })
+
+    const parseResult = validateTroubleshootingMetadata(frontmatter)
+    if ('error' in parseResult) {
+      console.error(
+        `Error validating troubleshooting metadata\nEntry:%O\nError:%O`,
+        frontmatter,
+        parseResult.error
+      )
+      return null
+    }
+
+    const mdxTree = fromMarkdown(content, {
+      extensions: [gfm(), mdxjs()],
+      mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()],
+    })
+    visit(mdxTree, (node) => {
+      if ('children' in node) {
+        node.children = node.children.filter(
+          (child) =>
+            ![
+              'mdxJsxFlowExpression',
+              'mdxJsxTextExpression',
+              'mdxFlowExpression',
+              'mdxTextExpression',
+              'mdxJsxFlowElement',
+              'mdxJsxTextElement',
+              'mdxJsxExpressionAttribute',
+              'mdxJsxAttribute',
+              'mdxJsxAttributeValueExpression',
+              'mdxjsEsm',
+            ].includes(child.type)
+        )
+      }
+    })
+    const contentWithoutJsx = toMarkdown(mdxTree, {
+      extensions: [gfmToMarkdown()],
+    })
+
+    return {
+      filePath,
+      content,
+      contentWithoutJsx,
+      data: parseResult.data,
+    }
+  })
+
+  return (await Promise.all(troubleshootingFiles)).filter(Boolean)
+}
+
+/**
+ * @param {TroubleshootingMetadata} meta
+ */
+export function getArticleSlug(meta) {
+  const slugifiedTitle = meta.title.toLowerCase().replace(/\s+/g, '-')
+  const escapedTitle = encodeURIComponent(slugifiedTitle)
+  return escapedTitle
+}

apps/docs/features/docs/Troubleshooting.utils.ts

@@ -1,137 +1,38 @@
-import matter from 'gray-matter'
-import { fromMarkdown } from 'mdast-util-from-markdown'
-import { mdxFromMarkdown } from 'mdast-util-mdx'
-import { toMarkdown } from 'mdast-util-to-markdown'
-import { mdxjs } from 'micromark-extension-mdxjs'
-import { readdir, readFile, stat } from 'node:fs/promises'
-import { join } from 'node:path'
-import toml from 'toml'
-import { visit } from 'unist-util-visit'
-import { v4 as uuidv4 } from 'uuid'
 import { z } from 'zod'
 
-import { DOCS_DIRECTORY } from 'lib/docs'
+import { supabaseAdmin } from '~/lib/supabaseAdmin'
 import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs'
+import {
+  getAllTroubleshootingEntriesInternal,
+  getArticleSlug as getArticleSlugInternal,
+  TroubleshootingSchema,
+  TROUBLESHOOTING_DIRECTORY,
+} from './Troubleshooting.utils.common.mjs'
 import { formatError } from './Troubleshooting.utils.shared'
 
-const TROUBLESHOOTING_DIRECTORY = join(DOCS_DIRECTORY, 'content/troubleshooting')
+// We don't have strictNullChecks on because we still have too many violations
+// in the code base, so required types aren't typed properly by Zod.
+export type ITroubleshootingMetadata = z.infer<typeof TroubleshootingSchema> &
+  Pick<Required<z.infer<typeof TroubleshootingSchema>>, 'title' | 'topics' | 'database_id'>
 
-const TroubleshootingSchema = z
-  .object({
-    title: z.string(),
-    topics: z.array(
-      z.enum([
-        'ai',
-        'auth',
-        'branching',
-        'cli',
-        'database',
-        'functions',
-        'platform',
-        'realtime',
-        'self-hosting',
-        'storage',
-        'studio',
-        'supavisor',
-        'terraform',
-      ])
-    ),
-    keywords: z.array(z.string()).optional(),
-    api: z
-      .object({
-        sdk: z.array(z.string()).optional(),
-        management_api: z.array(z.string()).optional(),
-        cli: z.array(z.string()).optional(),
-      })
-      .strict()
-      .optional(),
-    errors: z
-      .array(
-        z
-          .object({
-            http_status_code: z.number().optional(),
-            code: z.string().optional(),
-            message: z.string().optional(),
-          })
-          .strict()
-      )
-      .optional(),
-    database_id: z.string().uuid().default(uuidv4()),
-    created_at: z.date({ coerce: true }).optional(),
-    updated_at: z.date({ coerce: true }).optional(),
-  })
-  .strict()
-
-export type ITroubleshootingMetadata = z.infer<typeof TroubleshootingSchema>
-
-const validateTroubleshootingMetadata = (troubleshootingMetadata: unknown) => {
-  return TroubleshootingSchema.safeParse(troubleshootingMetadata)
+export interface ITroubleshootingEntry {
+  filePath: string
+  content: string
+  contentWithoutJsx: string
+  data: ITroubleshootingMetadata
 }
 
-async function getAllTroubleshootingEntriesInternal() {
-  const troubleshootingDirectoryContents = await readdir(TROUBLESHOOTING_DIRECTORY, {
-    recursive: true,
-  })
-  const troubleshootingFiles = troubleshootingDirectoryContents.map(async (entry) => {
-    const isHidden = entry.startsWith('_')
-    if (isHidden) return null
+export const getArticleSlug = getArticleSlugInternal
 
-    const isFile = (await stat(join(TROUBLESHOOTING_DIRECTORY, entry))).isFile()
-    if (!isFile) return null
-
-    const fileContents = await readFile(join(TROUBLESHOOTING_DIRECTORY, entry), 'utf-8')
-    const { content, data: frontmatter } = matter(fileContents, {
-      language: 'toml',
-      engines: { toml: toml.parse.bind(toml) },
-    })
-
-    const parseResult = validateTroubleshootingMetadata(frontmatter)
-    if ('error' in parseResult) {
-      console.error(`Error validating troubleshooting metadata for ${entry}:`, parseResult.error)
-      return null
-    }
-
-    const mdxTree = fromMarkdown(content, {
-      extensions: [mdxjs()],
-      mdastExtensions: [mdxFromMarkdown()],
-    })
-    visit(mdxTree, (node) => {
-      if ('children' in node) {
-        node.children = node.children.filter(
-          (child) =>
-            ![
-              'mdxJsxFlowExpression',
-              'mdxJsxTextExpression',
-              'mdxFlowExpression',
-              'mdxTextExpression',
-              'mdxJsxFlowElement',
-              'mdxJsxTextElement',
-              'mdxJsxExpressionAttribute',
-              'mdxJsxAttribute',
-              'mdxJsxAttributeValueExpression',
-              'mdxjsEsm',
-            ].includes(child.type)
-        )
-      }
-    })
-    const contentWithoutJsx = toMarkdown(mdxTree)
-
-    return {
-      content,
-      contentWithoutJsx,
-      data: parseResult.data,
-    }
-  })
-
-  return (await Promise.all(troubleshootingFiles)).filter(Boolean)
+async function getAllTroubleshootingEntriesTyped() {
+  const result = (await getAllTroubleshootingEntriesInternal()) as ITroubleshootingEntry[]
+  return result
 }
-
 export const getAllTroubleshootingEntries = cache_fullProcess_withDevCacheBust(
-  getAllTroubleshootingEntriesInternal,
+  getAllTroubleshootingEntriesTyped,
   TROUBLESHOOTING_DIRECTORY,
   () => JSON.stringify([])
 )
-export type ITroubleshootingEntry = Awaited<ReturnType<typeof getAllTroubleshootingEntries>>[number]
 
 export async function getAllTroubleshootingKeywords() {
   const entries = await getAllTroubleshootingEntries()
@@ -185,8 +86,26 @@ export async function getAllTroubleshootingErrors() {
   return Array.from(allErrors).sort(sortErrors)
 }
 
-export function getArticleSlug(entry: ITroubleshootingMetadata) {
-  const slugifiedTitle = entry.title.toLowerCase().replace(/\s+/g, '-')
-  const escapedTitle = encodeURIComponent(slugifiedTitle)
-  return escapedTitle
+async function getTroubleshootingUpdatedDatesInternal() {
+  const databaseIds = (await getAllTroubleshootingEntries())
+    .map((entry) => entry.data.database_id)
+    .filter((id) => !id.startsWith('pseudo-'))
+
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('id, date_updated')
+    .in('id', databaseIds)
+  if (error) {
+    console.error(error)
+  }
+
+  return (data ?? []).reduce((acc, entry) => {
+    acc.set(entry.id, new Date(entry.date_updated))
+    return acc
+  }, new Map<string, Date>())
 }
+export const getTroubleshootingUpdatedDates = cache_fullProcess_withDevCacheBust(
+  getTroubleshootingUpdatedDatesInternal,
+  TROUBLESHOOTING_DIRECTORY,
+  () => JSON.stringify([])
+)

apps/docs/lib/supabaseAdmin.ts

@@ -0,0 +1,18 @@
+import 'server-only'
+
+import { createClient, type SupabaseClient } from '@supabase/supabase-js'
+
+import { type Database } from 'common'
+
+let supabaseAdminClient: SupabaseClient<Database> | null = null
+
+export function supabaseAdmin() {
+  if (!supabaseAdminClient) {
+    supabaseAdminClient = createClient(
+      process.env.NEXT_PUBLIC_SUPABASE_URL,
+      process.env.SUPABASE_SECRET_KEY
+    )
+  }
+
+  return supabaseAdminClient
+}

apps/docs/package.json

@@ -18,6 +18,7 @@
     "test:smoke": "npm run codegen:references && vitest -t \"prod smoke test\"",
     "embeddings": "tsx scripts/search/generate-embeddings.ts",
     "embeddings:refresh": "npm run embeddings -- --refresh",
+    "troubleshooting:sync": "node features/docs/Troubleshooting.script.mjs",
     "last-changed": "tsx scripts/last-changed.ts",
     "last-changed:reset": "npm run last-changed -- --reset",
     "codegen:references": "tsx features/docs/Reference.generated.script.ts",
@@ -127,6 +128,7 @@
     "openapi-types": "^12.1.3",
     "simple-git": "^3.24.0",
     "slugify": "^1.6.6",
+    "smol-toml": "^1.3.0",
     "tsconfig": "*",
     "tsx": "^4.6.2",
     "typescript": "~5.5.0",

package-lock.json

@@ -958,6 +958,7 @@
         "openapi-types": "^12.1.3",
         "simple-git": "^3.24.0",
         "slugify": "^1.6.6",
+        "smol-toml": "^1.3.0",
         "tsconfig": "*",
         "tsx": "^4.6.2",
         "typescript": "~5.5.0",
@@ -39408,6 +39409,18 @@
         "npm": ">= 3.0.0"
       }
     },
+    "node_modules/smol-toml": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.0.tgz",
+      "integrity": "sha512-tWpi2TsODPScmi48b/OQZGi2lgUmBCHy6SZrhi/FdnnHiU1GwebbCfuQuxsC3nHaLwtYeJGPrDZDIeodDOc4pA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/cyyynthia"
+      }
+    },
     "node_modules/snake-case": {
       "version": "3.0.4",
       "dev": true,

packages/common/database-types.ts

@@ -318,9 +318,11 @@ export type Database = {
       troubleshooting_entries: {
         Row: {
           api: Json | null
+          checksum: string
           date_created: string
           date_updated: string
           errors: Json[] | null
+          github_id: string
           github_url: string
           id: string
           keywords: string[] | null
@@ -329,9 +331,11 @@ export type Database = {
         }
         Insert: {
           api?: Json | null
+          checksum: string
           date_created?: string
           date_updated?: string
           errors?: Json[] | null
+          github_id: string
           github_url: string
           id?: string
           keywords?: string[] | null
@@ -340,9 +344,11 @@ export type Database = {
         }
         Update: {
           api?: Json | null
+          checksum?: string
           date_created?: string
           date_updated?: string
           errors?: Json[] | null
+          github_id?: string
           github_url?: string
           id?: string
           keywords?: string[] | null
@@ -464,6 +470,33 @@ export type Database = {
         }
         Returns: unknown
       }
+      json_matches_schema: {
+        Args: {
+          schema: Json
+          instance: Json
+        }
+        Returns: boolean
+      }
+      jsonb_matches_schema: {
+        Args: {
+          schema: Json
+          instance: Json
+        }
+        Returns: boolean
+      }
+      jsonschema_is_valid: {
+        Args: {
+          schema: Json
+        }
+        Returns: boolean
+      }
+      jsonschema_validation_errors: {
+        Args: {
+          schema: Json
+          instance: Json
+        }
+        Returns: string[]
+      }
       match_page_sections_v2: {
         Args: {
           embedding: string
@@ -491,6 +524,12 @@ export type Database = {
         }
         Returns: string
       }
+      validate_troubleshooting_errors: {
+        Args: {
+          errors: Json[]
+        }
+        Returns: boolean
+      }
       vector_avg: {
         Args: {
           "": number[]