diff --git a/apps/docs/DEVELOPERS.md b/apps/docs/DEVELOPERS.md index 5c5b74e39de..a998eaba7d2 100644 --- a/apps/docs/DEVELOPERS.md +++ b/apps/docs/DEVELOPERS.md @@ -96,4 +96,6 @@ On the Next.JS side of things, these work almost exactly the same as the client #### Search -Search is handled through Algolia. When the site is built, a [search script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/build-search.ts) runs through all of the types of content, generating search objects that are sent to Algolia to index. +Search is handled using a Supabase instance. During CI, [a script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/search/generate-embeddings.ts) aggregates all content sources (eg. guides, reference docs, etc), indexes them using OpenAI embeddings, and stores them in a Supabase database. + +At runtime, an [Edge Function](https://github.com/supabase/supabase/blob/master/supabase/functions) is executed that performs a similarity search between the user's query and the above content sources using [`pgvector`](https://github.com/pgvector/pgvector) embeddings. diff --git a/apps/docs/layouts/SiteLayout.tsx b/apps/docs/layouts/SiteLayout.tsx index 1c0c1182cb6..08cb34e0835 100644 --- a/apps/docs/layouts/SiteLayout.tsx +++ b/apps/docs/layouts/SiteLayout.tsx @@ -8,6 +8,7 @@ import { memo, useEffect } from 'react' import Footer from '~/components/Navigation/Footer' import { menuState, useMenuLevelId, useMenuMobileOpen } from '~/hooks/useMenuState' import Head from 'next/head' +import { Announcement, AnnouncementCountdown } from 'ui' const levelsData = { home: { @@ -324,6 +325,11 @@ const SiteLayout = ({ children }) => { Supabase Docs
+
+ + + +
diff --git a/apps/docs/package.json b/apps/docs/package.json index cfda86a3831..ca12c92170b 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -9,9 +9,9 @@ "start": "next start", "lint": "next lint", "build:sitemap": "node ./internals/generate-sitemap.mjs", - "embeddings": "tsx scripts/generate-embeddings.ts", + "embeddings": "tsx scripts/search/generate-embeddings.ts", "embeddings:refresh": "npm run embeddings -- --refresh", - "postbuild": "ts-node ./scripts/build-search.ts && node ./internals/generate-sitemap.mjs", + "postbuild": "node ./internals/generate-sitemap.mjs", "generate:all": "npm-run-all --parallel gen:api gen:cli gen:gotrue gen:storage gen:supabase-dart:v0 gen:supabase-dart:v1 gen:supabase-csharp:v0 gen:supabase-js:v1 gen:supabase-js:v2 gen:realtime", "gen:api": "npm-run-all gen:api:usage", "gen:api:usage": "ts-node ./generator/index.ts gen --type api --url https://api.supabase.com --input ../../spec/transforms/api_v0_openapi_deparsed.json --output ./docs/reference/api/generated/usage.mdx", diff --git a/apps/docs/scripts/build-search.ts b/apps/docs/scripts/build-search.ts deleted file mode 100644 index 1fdc4e18a88..00000000000 --- a/apps/docs/scripts/build-search.ts +++ /dev/null @@ -1,155 +0,0 @@ -import fs from 'fs' -import crypto from 'crypto' -import path from 'path' -import matter from 'gray-matter' -import dotenv from 'dotenv' -import algoliasearch from 'algoliasearch/lite' -import { isEmpty } from 'lodash' - -// search objects -import { generateClientLibSearchObjects } from './files/client-libs' -import { generateAPISearchObjects } from './files/api' -import { generateCLISearchObjects } from './files/cli' - -const cliObjects = generateCLISearchObjects() -const apiObjects = generateAPISearchObjects() -const clientLibSearchObjects = generateClientLibSearchObjects() - -// @ts-ignore -// The properties of the searchObject that are specifically read -// by DocSearch are "type" and "hierarchy". The rest, though saved into Algolia (which we -// can potentially use to craft more nuanced search experiences) are not used by DocSearch -const ignoredFiles = [ - 'pages/404.mdx', - 'pages/faq.mdx', - 'pages/support.mdx', - 'pages/oss.tsx', - 'pages/_app.tsx', - 'pages/_document.tsx', - 'pages/[...slug].tsx', - 'pages/handbook/contributing.mdx', - 'pages/handbook/introduction.mdx', - 'pages/handbook/supasquad.mdx', -] - -async function walk(dir) { - let files = await fs.promises.readdir(dir) - //@ts-ignore - files = await Promise.all( - files.map(async (file) => { - const filePath = path.join(dir, file) - const stats = await fs.promises.stat(filePath) - if (stats.isDirectory()) return walk(filePath) - else if (stats.isFile()) return filePath - }) - ) - - return files.reduce((all, folderContents) => all.concat(folderContents), []) -} - -;(async function () { - // initialize environment variables - dotenv.config() - - if (!process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || !process.env.ALGOLIA_SEARCH_ADMIN_KEY) { - return console.log( - 'Missing Algolia app ID / admin Key: skipping saving of Algolia search index' - ) - } - - console.log('Preparing docs indexing for Algolia') - - try { - const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME - const client = algoliasearch( - process.env.NEXT_PUBLIC_ALGOLIA_APP_ID, - process.env.ALGOLIA_SEARCH_ADMIN_KEY - ) - const index = client.initIndex(indexName) - - const guidePages = (await walk('pages')).filter((slug) => !ignoredFiles.includes(slug)) - - // generate search objects for mdx guide pages - const guidePagesearchObjects = guidePages - .map((slug) => { - let id, title, description - const fileContents = fs.readFileSync(slug, 'utf8') - const { data, content } = matter(fileContents) - - if (isEmpty(data)) { - // Guide pages do not have front-matter meta, unlike reference pages, have to manually extract - const metaIndex = fileContents.indexOf('export const meta = {') - if (metaIndex !== -1) { - const metaString = - fileContents - .slice(metaIndex + 20, fileContents.indexOf('}', metaIndex + 1) + 1) - .replace(/\n/g, '') - .slice(0, -2) + '}' - const meta = eval(`(${metaString})`) - id = meta.id - title = meta.title - description = meta.description - } - } else { - id = data.id - title = data.title - description = data.description - } - - const url = (slug.includes('/generated') ? slug.replace('/generated', '') : slug) - .replace('docs', '') - .replace('pages', '') - .replace(/\.mdx$/, '') - const source = slug.includes('/reference') ? 'reference' : 'guide' - - const object = { - // For Algolia - objectID: crypto.randomUUID(), - id, - title, - description, - url, - source, - //pageContent: content, - pageContent: '', - category: undefined, - version: undefined, - - // Docsearch specific - type: 'lvl1', - hierarchy: { - lvl0: 'Guides', - lvl1: title, - lvl2: null, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - } - - return object - }) - // Some of the reference generated files come with an 'index' page that we can ignore - .filter((object) => !object.url.endsWith('/index')) - .filter((object) => !object.url.endsWith('/.gitkeep')) - - const combinedSearchObjects = guidePagesearchObjects.concat( - clientLibSearchObjects, - apiObjects, - cliObjects - ) - - //@ts-ignore - await index.clearObjects() - console.log(`Successfully cleared records from ${indexName}`) - - //@ts-ignore - const algoliaResponse = await index.saveObjects(combinedSearchObjects) - - //@ts-ignore - console.log(`Successfully saved ${algoliaResponse.objectIDs.length} records into ${indexName}.`) - } catch (error) { - console.log('Error:', error) - } -})() diff --git a/apps/docs/scripts/files/api.ts b/apps/docs/scripts/files/api.ts deleted file mode 100644 index ed98a9c4921..00000000000 --- a/apps/docs/scripts/files/api.ts +++ /dev/null @@ -1,43 +0,0 @@ -import crypto from 'crypto' -import { flattenSections } from '../../lib/helpers' - -import apiCommonSections from '~/../../spec/common-api-sections.json' -import specFile from '~/../../spec/transforms/api_v0_openapi_deparsed.json' -import { gen_v3 } from '../../lib/refGenerator/helpers' - -// @ts-ignore -const generatedSpec = gen_v3(specFile, 'wat', { apiUrl: 'apiv0' }) -const sections = flattenSections(apiCommonSections) - -export function generateAPISearchObjects() { - let searchObjects = [] - - //@ts-ignore - sections.map((section) => { - const object = searchObjects.push({ - objectID: crypto.randomUUID(), - id: section.id, - title: section.title, - // @ts-ignore - description: generatedSpec.operations.find((item) => item.operationId === section.id) - ?.summary, - url: `/reference/api/${section.slug}`, - source: 'reference', - pageContent: '', - category: section.product, - version: '', - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: 'Management API', - lvl2: section.title, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - }) - return searchObjects -} diff --git a/apps/docs/scripts/files/cli.ts b/apps/docs/scripts/files/cli.ts deleted file mode 100644 index 84caae225c6..00000000000 --- a/apps/docs/scripts/files/cli.ts +++ /dev/null @@ -1,43 +0,0 @@ -import crypto from 'crypto' -import fs from 'fs' -import { flattenSections } from '../../lib/helpers' -import cliCommonSections from '~/../../spec/common-cli-sections.json' -import yaml from 'js-yaml' - -// @ts-ignore - -const spec = yaml.load(fs.readFileSync(`../../spec/cli_v1_commands.yaml`, 'utf8')) - -const commonSections = flattenSections(cliCommonSections) - -export function generateCLISearchObjects() { - let searchObjects = [] - - //@ts-ignore - spec.commands.map((section) => { - const object = searchObjects.push({ - objectID: crypto.randomUUID(), - id: section.id, - title: section.title, - // @ts-ignore - description: section.description?.substr(0, section.description.indexOf('\n')), - url: `/reference/cli/${commonSections.find((item) => item.id === section.id)?.slug}`, - source: 'reference', - pageContent: '', - category: section.product, - version: '', - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: 'Supabase CLI', - lvl2: section.title, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - }) - return searchObjects -} diff --git a/apps/docs/scripts/files/client-libs.ts b/apps/docs/scripts/files/client-libs.ts deleted file mode 100644 index 714769ebcc7..00000000000 --- a/apps/docs/scripts/files/client-libs.ts +++ /dev/null @@ -1,57 +0,0 @@ -import fs from 'fs' -import crypto from 'crypto' -import yaml from 'js-yaml' -import { flattenSections } from '../../lib/helpers' -import { nameMap } from '../helpers' - -import commonLibSections from '~/../../spec/common-client-libs-sections.json' - -const clientLibFiles = [ - { fileName: 'supabase_js_v2', label: 'javascript', version: 'v2', versionSlug: false }, - { fileName: 'supabase_js_v1', label: 'javascript', version: 'v1', versionSlug: true }, - { fileName: 'supabase_dart_v1', label: 'dart', version: 'v1', versionSlug: false }, - { fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true }, - { fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true }, -] - -const flatCommonLibSections = flattenSections(commonLibSections) - -export function generateClientLibSearchObjects() { - // loop through each spec file, find the correspending entry in the common file and grab the title / description / slug - let clientLibSearchObjects = [] - - clientLibFiles.map((file) => { - const specs = yaml.load(fs.readFileSync(`../../spec/${file.fileName}.yml`, 'utf8')) - - //take each function id, find it in the commonLibSections file and return { id, title, slug, description, } - //@ts-ignore - specs.functions.map((fn) => { - const item = flatCommonLibSections.find((section) => section.id === fn.id) - if (item) { - const object = clientLibSearchObjects.push({ - objectID: crypto.randomUUID(), - id: item.id, - title: item.title, - description: item.title, - url: `/reference/${file.label}/${file.versionSlug ? file.version + '/' : ''}${item.slug}`, - source: 'reference', - pageContent: '', - category: item.product, - version: file.version, - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: `${nameMap[file.label]} ${file.version}`, - lvl2: item.title, - lvl3: file.version, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - } - }) - }) - return clientLibSearchObjects -} diff --git a/apps/docs/scripts/generate-embeddings.ts b/apps/docs/scripts/generate-embeddings.ts deleted file mode 100644 index 520450083e3..00000000000 --- a/apps/docs/scripts/generate-embeddings.ts +++ /dev/null @@ -1,659 +0,0 @@ -import { createClient } from '@supabase/supabase-js' -import { createHash } from 'crypto' -import dotenv from 'dotenv' -import { ObjectExpression } from 'estree' -import { readdir, readFile, stat } from 'fs/promises' -import GithubSlugger from 'github-slugger' -import yaml from 'js-yaml' -import { Content, Root } from 'mdast' -import { fromMarkdown } from 'mdast-util-from-markdown' -import { mdxFromMarkdown, MdxjsEsm } from 'mdast-util-mdx' -import { toMarkdown } from 'mdast-util-to-markdown' -import { toString } from 'mdast-util-to-string' -import { mdxjs } from 'micromark-extension-mdxjs' -import 'openai' -import { Configuration, OpenAIApi } from 'openai' -import { OpenAPIV3 } from 'openapi-types' -import { basename, dirname, join } from 'path' -import { u } from 'unist-builder' -import { filter } from 'unist-util-filter' -import { inspect } from 'util' -import { ICommonFunc, IFunctionDefinition, ISpec } from '../components/reference/Reference.types' -import { CliCommand, CliSpec } from '../generator/types/CliSpec' -import { flattenSections } from '../lib/helpers' -import { enrichedOperation, gen_v3 } from '../lib/refGenerator/helpers' - -dotenv.config() - -const ignoredFiles = ['pages/404.mdx'] - -/** - * Extracts ES literals from an `estree` `ObjectExpression` - * into a plain JavaScript object. - */ -function getObjectFromExpression(node: ObjectExpression) { - return node.properties.reduce< - Record - >((object, property) => { - if (property.type !== 'Property') { - return object - } - - const key = (property.key.type === 'Identifier' && property.key.name) || undefined - const value = (property.value.type === 'Literal' && property.value.value) || undefined - - if (!key) { - return object - } - - return { - ...object, - [key]: value, - } - }, {}) -} - -/** - * Extracts the `meta` ESM export from the MDX file. - * - * This info is akin to frontmatter. - */ -function extractMetaExport(mdxTree: Root) { - const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => { - return ( - node.type === 'mdxjsEsm' && - node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' && - node.data.estree.body[0].declaration?.type === 'VariableDeclaration' && - node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' && - node.data.estree.body[0].declaration.declarations[0].id.name === 'meta' - ) - }) - - if (!metaExportNode) { - return undefined - } - - const objectExpression = - (metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' && - metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' && - metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' && - metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' && - metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type === - 'ObjectExpression' && - metaExportNode.data.estree.body[0].declaration.declarations[0].init) || - undefined - - if (!objectExpression) { - return undefined - } - - return getObjectFromExpression(objectExpression) -} - -/** - * Splits a `mdast` tree into multiple trees based on - * a predicate function. Will include the splitting node - * at the beginning of each tree. - * - * Useful to split a markdown file into smaller sections. - */ -function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) { - return tree.children.reduce((trees, node) => { - const [lastTree] = trees.slice(-1) - - if (!lastTree || predicate(node)) { - const tree: Root = u('root', [node]) - return trees.concat(tree) - } - - lastTree.children.push(node) - return trees - }, []) -} - -type Meta = ReturnType - -type Section = { - content: string - heading?: string - slug?: string -} - -type ProcessedMdx = { - checksum: string - meta: Meta - sections: Section[] -} - -/** - * Processes MDX content for search indexing. - * It extracts metadata, strips it of all JSX, - * and splits it into sub-sections based on criteria. - */ -function processMdxForSearch(content: string): ProcessedMdx { - const checksum = createHash('sha256').update(content).digest('base64') - - const mdxTree = fromMarkdown(content, { - extensions: [mdxjs()], - mdastExtensions: [mdxFromMarkdown()], - }) - - const meta = extractMetaExport(mdxTree) - - // Remove all MDX elements from markdown - const mdTree = filter( - mdxTree, - (node) => - ![ - 'mdxjsEsm', - 'mdxJsxFlowElement', - 'mdxJsxTextElement', - 'mdxFlowExpression', - 'mdxTextExpression', - ].includes(node.type) - ) - - if (!mdTree) { - return { - checksum, - meta, - sections: [], - } - } - - const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading') - - const slugger = new GithubSlugger() - - const sections = sectionTrees.map((tree) => { - const [firstNode] = tree.children - - const heading = firstNode.type === 'heading' ? toString(firstNode) : undefined - const slug = heading ? slugger.slug(heading) : undefined - - return { - content: toMarkdown(tree), - heading, - slug, - } - }) - - return { - checksum, - meta, - sections, - } -} - -type WalkEntry = { - path: string - parentPath?: string -} - -async function walk(dir: string, parentPath?: string): Promise { - const immediateFiles = await readdir(dir) - - const recursiveFiles = await Promise.all( - immediateFiles.map(async (file) => { - const path = join(dir, file) - const stats = await stat(path) - if (stats.isDirectory()) { - // Keep track of document hierarchy (if this dir has corresponding doc file) - const docPath = `${basename(path)}.mdx` - - return walk( - path, - immediateFiles.includes(docPath) ? join(dirname(path), docPath) : parentPath - ) - } else if (stats.isFile()) { - return [ - { - path: path, - parentPath, - }, - ] - } else { - return [] - } - }) - ) - - const flattenedFiles = recursiveFiles.reduce( - (all, folderContents) => all.concat(folderContents), - [] - ) - - return flattenedFiles.sort((a, b) => a.path.localeCompare(b.path)) -} - -abstract class BaseEmbeddingSource { - checksum?: string - meta?: Meta - sections?: Section[] - - constructor(public source: string, public path: string, public parentPath?: string) {} - - abstract load(): Promise<{ checksum: string; meta?: Meta; sections: Section[] }> -} - -class MarkdownEmbeddingSource extends BaseEmbeddingSource { - type: 'markdown' = 'markdown' - - constructor(source: string, public filePath: string, public parentFilePath?: string) { - const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '') - const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '') - - super(source, path, parentPath) - } - - async load() { - const contents = await readFile(this.filePath, 'utf8') - - const { checksum, meta, sections } = processMdxForSearch(contents) - - this.checksum = checksum - this.meta = meta - this.sections = sections - - return { - checksum, - meta, - sections, - } - } -} - -abstract class ReferenceEmbeddingSource extends BaseEmbeddingSource { - type: 'reference' = 'reference' - - constructor( - source: string, - path: string, - public meta: Meta, - public specFilePath: string, - public sectionsFilePath: string - ) { - super(source, path) - } - - async load() { - const specContents = await readFile(this.specFilePath, 'utf8') - const refSectionsContents = await readFile(this.sectionsFilePath, 'utf8') - - const refSections: ICommonFunc[] = JSON.parse(refSectionsContents) - const flattenedRefSections = flattenSections(refSections) - - const checksum = createHash('sha256') - .update(specContents + refSectionsContents) - .digest('base64') - - const specSections = this.getSpecSections(specContents) - - const sections = flattenedRefSections - .map((refSection) => { - const specSection = this.matchSpecSection(specSections, refSection.id) - - if (!specSection) { - return - } - - return { - heading: refSection.title, - slug: refSection.slug, - content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection( - specSection, - refSection - )}`, - } - }) - .filter((section) => !!section) - - this.checksum = checksum - this.sections = sections - - return { - checksum, - sections, - meta: this.meta, - } - } - - abstract getSpecSections(specContents: string): SpecSection[] - abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection - abstract formatSection(specSection: SpecSection, refSection: ICommonFunc): string -} - -class OpenApiEmbeddingSource extends ReferenceEmbeddingSource { - getSpecSections(specContents: string): enrichedOperation[] { - const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents) - - const generatedSpec = gen_v3(spec, '', { - apiUrl: 'apiv0', - }) - - return generatedSpec.operations - } - matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation { - return operations.find((operation) => operation.operationId === id) - } - formatSection(specOperation: enrichedOperation) { - const { summary, description, operation, path, tags } = specOperation - return JSON.stringify({ - summary, - description, - operation, - path, - tags, - }) - } -} - -class ClientLibEmbeddingSource extends ReferenceEmbeddingSource { - getSpecSections(specContents: string): IFunctionDefinition[] { - const spec = yaml.load(specContents) as ISpec - - return spec.functions - } - matchSpecSection(functionDefinitions: IFunctionDefinition[], id: string): IFunctionDefinition { - return functionDefinitions.find((functionDefinition) => functionDefinition.id === id) - } - formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonFunc): string { - const { title } = refSection - const { description, title: functionName } = functionDefinition - - return JSON.stringify({ - title, - description, - functionName, - }) - } -} - -class CliEmbeddingSource extends ReferenceEmbeddingSource { - getSpecSections(specContents: string): CliCommand[] { - const spec = yaml.load(specContents) as CliSpec - - return spec.commands - } - matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand { - return cliCommands.find((cliCommand) => cliCommand.id === id) - } - formatSection(cliCommand: CliCommand): string { - const { summary, description, usage } = cliCommand - return JSON.stringify({ - summary, - description, - usage, - }) - } -} - -type EmbeddingSource = - | MarkdownEmbeddingSource - | OpenApiEmbeddingSource - | ClientLibEmbeddingSource - | CliEmbeddingSource - -async function generateEmbeddings() { - // TODO: use better CLI lib like yargs - const args = process.argv.slice(2) - const shouldRefresh = args.includes('--refresh') - - if ( - !process.env.NEXT_PUBLIC_SUPABASE_URL || - !process.env.SUPABASE_SERVICE_ROLE_KEY || - !process.env.OPENAI_KEY - ) { - return console.log( - 'Environment variables NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, and OPENAI_KEY are required: skipping embeddings generation' - ) - } - - const supabaseClient = createClient( - process.env.NEXT_PUBLIC_SUPABASE_URL, - process.env.SUPABASE_SERVICE_ROLE_KEY, - { - auth: { - persistSession: false, - autoRefreshToken: false, - }, - } - ) - - const embeddingSources: EmbeddingSource[] = [ - new OpenApiEmbeddingSource( - 'api', - '/reference/api', - { title: 'Management API Reference' }, - '../../spec/transforms/api_v0_openapi_deparsed.json', - '../../spec/common-api-sections.json' - ), - new ClientLibEmbeddingSource( - 'js-lib', - '/reference/javascript', - { title: 'JavaScript Reference' }, - '../../spec/supabase_js_v2.yml', - '../../spec/common-client-libs-sections.json' - ), - new ClientLibEmbeddingSource( - 'dart-lib', - '/reference/dart', - { title: 'Dart Reference' }, - '../../spec/supabase_dart_v1.yml', - '../../spec/common-client-libs-sections.json' - ), - new ClientLibEmbeddingSource( - 'python-lib', - '/reference/python', - { title: 'Python Reference' }, - '../../spec/supabase_py_v2.yml', - '../../spec/common-client-libs-sections.json' - ), - new ClientLibEmbeddingSource( - 'csharp-lib', - '/reference/csharp', - { title: 'C# Reference' }, - '../../spec/supabase_csharp_v0.yml', - '../../spec/common-client-libs-sections.json' - ), - new CliEmbeddingSource( - 'cli', - '/reference/cli', - { title: 'CLI Reference' }, - '../../spec/cli_v1_commands.yaml', - '../../spec/common-cli-sections.json' - ), - ...(await walk('pages')) - .filter(({ path }) => /\.mdx?$/.test(path)) - .filter(({ path }) => !ignoredFiles.includes(path)) - .map((entry) => new MarkdownEmbeddingSource('guide', entry.path)), - ] - - console.log(`Discovered ${embeddingSources.length} pages`) - - if (!shouldRefresh) { - console.log('Checking which pages are new or have changed') - } else { - console.log('Refresh flag set, re-generating all pages') - } - - for (const embeddingSource of embeddingSources) { - const { type, source, path, parentPath } = embeddingSource - - try { - const { checksum, meta, sections } = await embeddingSource.load() - - // Check for existing page in DB and compare checksums - const { error: fetchPageError, data: existingPage } = await supabaseClient - .from('page') - .select('id, path, checksum, parentPage:parent_page_id(id, path)') - .filter('path', 'eq', path) - .limit(1) - .maybeSingle() - - if (fetchPageError) { - throw fetchPageError - } - - type Singular = T extends any[] ? undefined : T - - // We use checksum to determine if this page & its sections need to be regenerated - if (!shouldRefresh && existingPage?.checksum === checksum) { - const existingParentPage = existingPage?.parentPage as Singular< - typeof existingPage.parentPage - > - - // If parent page changed, update it - if (existingParentPage?.path !== parentPath) { - console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`) - const { error: fetchParentPageError, data: parentPage } = await supabaseClient - .from('page') - .select() - .filter('path', 'eq', parentPath) - .limit(1) - .maybeSingle() - - if (fetchParentPageError) { - throw fetchParentPageError - } - - const { error: updatePageError } = await supabaseClient - .from('page') - .update({ parent_page_id: parentPage?.id }) - .filter('id', 'eq', existingPage.id) - - if (updatePageError) { - throw updatePageError - } - } - continue - } - - if (existingPage) { - if (!shouldRefresh) { - console.log( - `[${path}] Docs have changed, removing old page sections and their embeddings` - ) - } else { - console.log(`[${path}] Refresh flag set, removing old page sections and their embeddings`) - } - - const { error: deletePageSectionError } = await supabaseClient - .from('page_section') - .delete() - .filter('page_id', 'eq', existingPage.id) - - if (deletePageSectionError) { - throw deletePageSectionError - } - } - - const { error: fetchParentPageError, data: parentPage } = await supabaseClient - .from('page') - .select() - .filter('path', 'eq', parentPath) - .limit(1) - .maybeSingle() - - if (fetchParentPageError) { - throw fetchParentPageError - } - - // Create/update page record. Intentionally clear checksum until we - // have successfully generated all page sections. - const { error: upsertPageError, data: page } = await supabaseClient - .from('page') - .upsert( - { - checksum: null, - path, - type, - source, - meta, - parent_page_id: parentPage?.id, - }, - { onConflict: 'path' } - ) - .select() - .limit(1) - .single() - - if (upsertPageError) { - throw upsertPageError - } - - console.log(`[${path}] Adding ${sections.length} page sections (with embeddings)`) - for (const { slug, heading, content } of sections) { - // OpenAI recommends replacing newlines with spaces for best results (specific to embeddings) - const input = content.replace(/\n/g, ' ') - - try { - const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY }) - const openai = new OpenAIApi(configuration) - - const embeddingResponse = await openai.createEmbedding({ - model: 'text-embedding-ada-002', - input, - }) - - if (embeddingResponse.status !== 200) { - throw new Error(inspect(embeddingResponse.data, false, 2)) - } - - const [responseData] = embeddingResponse.data.data - - const { error: insertPageSectionError, data: pageSection } = await supabaseClient - .from('page_section') - .insert({ - page_id: page.id, - slug, - heading, - content, - token_count: embeddingResponse.data.usage.total_tokens, - embedding: responseData.embedding, - }) - .select() - .limit(1) - .single() - - if (insertPageSectionError) { - throw insertPageSectionError - } - } catch (err) { - // TODO: decide how to better handle failed embeddings - console.error( - `Failed to generate embeddings for '${path}' page section starting with '${input.slice( - 0, - 40 - )}...'` - ) - - throw err - } - } - - // Set page checksum so that we know this page was stored successfully - const { error: updatePageError } = await supabaseClient - .from('page') - .update({ checksum }) - .filter('id', 'eq', page.id) - - if (updatePageError) { - throw updatePageError - } - } catch (err) { - console.error( - `Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.` - ) - console.error(err) - } - } - - console.log('Embedding generation complete') -} - -async function main() { - await generateEmbeddings() -} - -main().catch((err) => console.error(err)) diff --git a/apps/docs/scripts/helpers.ts b/apps/docs/scripts/helpers.ts deleted file mode 100644 index 97ae8ddb86e..00000000000 --- a/apps/docs/scripts/helpers.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const nameMap = { - api: 'Management API', - cli: 'Supabase CLI', - auth: 'Auth Server', - storage: 'Storage Server', - postgres: 'Postgres', - dart: 'Supabase Flutter Library', - javascript: 'Supabase JavaScript Library', -} diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts new file mode 100644 index 00000000000..d04765afc67 --- /dev/null +++ b/apps/docs/scripts/search/generate-embeddings.ts @@ -0,0 +1,225 @@ +import { createClient } from '@supabase/supabase-js' +import dotenv from 'dotenv' +import 'openai' +import { Configuration, OpenAIApi } from 'openai' +import { inspect } from 'util' +import { fetchSources } from './sources' + +dotenv.config() + +async function generateEmbeddings() { + // TODO: use better CLI lib like yargs + const args = process.argv.slice(2) + const shouldRefresh = args.includes('--refresh') + + const requiredEnvVars = ['NEXT_PUBLIC_SUPABASE_URL', 'SUPABASE_SERVICE_ROLE_KEY', 'OPENAI_KEY'] + + if (requiredEnvVars.some((name) => !process.env[name])) { + throw new Error( + `Environment variables ${requiredEnvVars.join( + ', ' + )} are required: skipping embeddings generation` + ) + } + + const supabaseClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY, + { + auth: { + persistSession: false, + autoRefreshToken: false, + }, + } + ) + + const embeddingSources = await fetchSources() + + console.log(`Discovered ${embeddingSources.length} pages`) + + if (!shouldRefresh) { + console.log('Checking which pages are new or have changed') + } else { + console.log('Refresh flag set, re-generating all pages') + } + + for (const embeddingSource of embeddingSources) { + const { type, source, path, parentPath } = embeddingSource + + try { + const { checksum, meta, sections } = await embeddingSource.load() + + // Check for existing page in DB and compare checksums + const { error: fetchPageError, data: existingPage } = await supabaseClient + .from('page') + .select('id, path, checksum, parentPage:parent_page_id(id, path)') + .filter('path', 'eq', path) + .limit(1) + .maybeSingle() + + if (fetchPageError) { + throw fetchPageError + } + + type Singular = T extends any[] ? undefined : T + + // We use checksum to determine if this page & its sections need to be regenerated + if (!shouldRefresh && existingPage?.checksum === checksum) { + const existingParentPage = existingPage?.parentPage as Singular< + typeof existingPage.parentPage + > + + // If parent page changed, update it + if (existingParentPage?.path !== parentPath) { + console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`) + const { error: fetchParentPageError, data: parentPage } = await supabaseClient + .from('page') + .select() + .filter('path', 'eq', parentPath) + .limit(1) + .maybeSingle() + + if (fetchParentPageError) { + throw fetchParentPageError + } + + const { error: updatePageError } = await supabaseClient + .from('page') + .update({ parent_page_id: parentPage?.id }) + .filter('id', 'eq', existingPage.id) + + if (updatePageError) { + throw updatePageError + } + } + continue + } + + if (existingPage) { + if (!shouldRefresh) { + console.log( + `[${path}] Docs have changed, removing old page sections and their embeddings` + ) + } else { + console.log(`[${path}] Refresh flag set, removing old page sections and their embeddings`) + } + + const { error: deletePageSectionError } = await supabaseClient + .from('page_section') + .delete() + .filter('page_id', 'eq', existingPage.id) + + if (deletePageSectionError) { + throw deletePageSectionError + } + } + + const { error: fetchParentPageError, data: parentPage } = await supabaseClient + .from('page') + .select() + .filter('path', 'eq', parentPath) + .limit(1) + .maybeSingle() + + if (fetchParentPageError) { + throw fetchParentPageError + } + + // Create/update page record. Intentionally clear checksum until we + // have successfully generated all page sections. + const { error: upsertPageError, data: page } = await supabaseClient + .from('page') + .upsert( + { + checksum: null, + path, + type, + source, + meta, + parent_page_id: parentPage?.id, + }, + { onConflict: 'path' } + ) + .select() + .limit(1) + .single() + + if (upsertPageError) { + throw upsertPageError + } + + console.log(`[${path}] Adding ${sections.length} page sections (with embeddings)`) + for (const { slug, heading, content } of sections) { + // OpenAI recommends replacing newlines with spaces for best results (specific to embeddings) + const input = content.replace(/\n/g, ' ') + + try { + const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY }) + const openai = new OpenAIApi(configuration) + + const embeddingResponse = await openai.createEmbedding({ + model: 'text-embedding-ada-002', + input, + }) + + if (embeddingResponse.status !== 200) { + throw new Error(inspect(embeddingResponse.data, false, 2)) + } + + const [responseData] = embeddingResponse.data.data + + const { error: insertPageSectionError, data: pageSection } = await supabaseClient + .from('page_section') + .insert({ + page_id: page.id, + slug, + heading, + content, + token_count: embeddingResponse.data.usage.total_tokens, + embedding: responseData.embedding, + }) + .select() + .limit(1) + .single() + + if (insertPageSectionError) { + throw insertPageSectionError + } + } catch (err) { + // TODO: decide how to better handle failed embeddings + console.error( + `Failed to generate embeddings for '${path}' page section starting with '${input.slice( + 0, + 40 + )}...'` + ) + + throw err + } + } + + // Set page checksum so that we know this page was stored successfully + const { error: updatePageError } = await supabaseClient + .from('page') + .update({ checksum }) + .filter('id', 'eq', page.id) + + if (updatePageError) { + throw updatePageError + } + } catch (err) { + console.error( + `Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.` + ) + console.error(err) + } + } + + console.log('Embedding generation complete') +} + +async function main() { + await generateEmbeddings() +} + +main().catch((err) => console.error(err)) diff --git a/apps/docs/scripts/search/sources/base.ts b/apps/docs/scripts/search/sources/base.ts new file mode 100644 index 00000000000..3a12513ed96 --- /dev/null +++ b/apps/docs/scripts/search/sources/base.ts @@ -0,0 +1,20 @@ +export type Json = Record< + string, + string | number | boolean | null | Json[] | { [key: string]: Json } +> + +export type Section = { + content: string + heading?: string + slug?: string +} + +export abstract class BaseSource { + checksum?: string + meta?: Json + sections?: Section[] + + constructor(public source: string, public path: string, public parentPath?: string) {} + + abstract load(): Promise<{ checksum: string; meta?: Json; sections: Section[] }> +} diff --git a/apps/docs/scripts/search/sources/index.ts b/apps/docs/scripts/search/sources/index.ts new file mode 100644 index 00000000000..66ff45b2932 --- /dev/null +++ b/apps/docs/scripts/search/sources/index.ts @@ -0,0 +1,85 @@ +import { MarkdownSource } from './markdown' +import { + CliReferenceSource, + ClientLibReferenceSource, + OpenApiReferenceSource, +} from './reference-doc' +import { walk } from './util' + +const ignoredFiles = ['pages/404.mdx'] + +export type SearchSource = + | MarkdownSource + | OpenApiReferenceSource + | ClientLibReferenceSource + | CliReferenceSource + +/** + * Fetches all the sources we want to index for search + */ +export async function fetchSources() { + const openApiReferenceSource = new OpenApiReferenceSource( + 'api', + '/reference/api', + { title: 'Management API Reference' }, + '../../spec/transforms/api_v0_openapi_deparsed.json', + '../../spec/common-api-sections.json' + ) + + const jsLibReferenceSource = new ClientLibReferenceSource( + 'js-lib', + '/reference/javascript', + { title: 'JavaScript Reference' }, + '../../spec/supabase_js_v2.yml', + '../../spec/common-client-libs-sections.json' + ) + + const dartLibReferenceSource = new ClientLibReferenceSource( + 'dart-lib', + '/reference/dart', + { title: 'Dart Reference' }, + '../../spec/supabase_dart_v1.yml', + '../../spec/common-client-libs-sections.json' + ) + + const pythonLibReferenceSource = new ClientLibReferenceSource( + 'python-lib', + '/reference/python', + { title: 'Python Reference' }, + '../../spec/supabase_py_v2.yml', + '../../spec/common-client-libs-sections.json' + ) + + const cSharpLibReferenceSource = new ClientLibReferenceSource( + 'csharp-lib', + '/reference/csharp', + { title: 'C# Reference' }, + '../../spec/supabase_csharp_v0.yml', + '../../spec/common-client-libs-sections.json' + ) + + const cliReferenceSource = new CliReferenceSource( + 'cli', + '/reference/cli', + { title: 'CLI Reference' }, + '../../spec/cli_v1_commands.yaml', + '../../spec/common-cli-sections.json' + ) + + const guideSources = (await walk('pages')) + .filter(({ path }) => /\.mdx?$/.test(path)) + .filter(({ path }) => !ignoredFiles.includes(path)) + .map((entry) => new MarkdownSource('guide', entry.path)) + + const sources: SearchSource[] = [ + openApiReferenceSource, + jsLibReferenceSource, + dartLibReferenceSource, + pythonLibReferenceSource, + cSharpLibReferenceSource, + cliReferenceSource, + ...guideSources, + ] + + return sources +} diff --git a/apps/docs/scripts/search/sources/markdown.ts b/apps/docs/scripts/search/sources/markdown.ts new file mode 100644 index 00000000000..1c7b29640c8 --- /dev/null +++ b/apps/docs/scripts/search/sources/markdown.ts @@ -0,0 +1,191 @@ +import { createHash } from 'crypto' +import { ObjectExpression } from 'estree' +import { readFile } from 'fs/promises' +import GithubSlugger from 'github-slugger' +import { Content, Root } from 'mdast' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { MdxjsEsm, mdxFromMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { toString } from 'mdast-util-to-string' +import { mdxjs } from 'micromark-extension-mdxjs' +import { u } from 'unist-builder' +import { filter } from 'unist-util-filter' +import { BaseSource, Json, Section } from './base' + +/** + * Extracts ES literals from an `estree` `ObjectExpression` + * into a plain JavaScript object. + */ +export function getObjectFromExpression(node: ObjectExpression) { + return node.properties.reduce< + Record + >((object, property) => { + if (property.type !== 'Property') { + return object + } + + const key = (property.key.type === 'Identifier' && property.key.name) || undefined + const value = (property.value.type === 'Literal' && property.value.value) || undefined + + if (!key) { + return object + } + + return { + ...object, + [key]: value, + } + }, {}) +} + +/** + * Extracts the `meta` ESM export from the MDX file. + * + * This info is akin to frontmatter. + */ +export function extractMetaExport(mdxTree: Root) { + const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => { + return ( + node.type === 'mdxjsEsm' && + node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' && + node.data.estree.body[0].declaration?.type === 'VariableDeclaration' && + node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' && + node.data.estree.body[0].declaration.declarations[0].id.name === 'meta' + ) + }) + + if (!metaExportNode) { + return undefined + } + + const objectExpression = + (metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' && + metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' && + metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' && + metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' && + metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type === + 'ObjectExpression' && + metaExportNode.data.estree.body[0].declaration.declarations[0].init) || + undefined + + if (!objectExpression) { + return undefined + } + + return getObjectFromExpression(objectExpression) +} + +/** + * Splits a `mdast` tree into multiple trees based on + * a predicate function. Will include the splitting node + * at the beginning of each tree. + * + * Useful to split a markdown file into smaller sections. + */ +export function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) { + return tree.children.reduce((trees, node) => { + const [lastTree] = trees.slice(-1) + + if (!lastTree || predicate(node)) { + const tree: Root = u('root', [node]) + return trees.concat(tree) + } + + lastTree.children.push(node) + return trees + }, []) +} + +/** + * Processes MDX content for search indexing. + * It extracts metadata, strips it of all JSX, + * and splits it into sub-sections based on criteria. + */ +export function processMdxForSearch(content: string): ProcessedMdx { + const checksum = createHash('sha256').update(content).digest('base64') + + const mdxTree = fromMarkdown(content, { + extensions: [mdxjs()], + mdastExtensions: [mdxFromMarkdown()], + }) + + const meta = extractMetaExport(mdxTree) + const serializableMeta: Json = JSON.parse(JSON.stringify(meta)) + + // Remove all MDX elements from markdown + const mdTree = filter( + mdxTree, + (node) => + ![ + 'mdxjsEsm', + 'mdxJsxFlowElement', + 'mdxJsxTextElement', + 'mdxFlowExpression', + 'mdxTextExpression', + ].includes(node.type) + ) + + if (!mdTree) { + return { + checksum, + meta: serializableMeta, + sections: [], + } + } + + const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading') + + const slugger = new GithubSlugger() + + const sections = sectionTrees.map((tree) => { + const [firstNode] = tree.children + + const heading = firstNode.type === 'heading' ? toString(firstNode) : undefined + const slug = heading ? slugger.slug(heading) : undefined + + return { + content: toMarkdown(tree), + heading, + slug, + } + }) + + return { + checksum, + meta: serializableMeta, + sections, + } +} + +export type ProcessedMdx = { + checksum: string + meta: Json + sections: Section[] +} + +export class MarkdownSource extends BaseSource { + type = 'markdown' as const + + constructor(source: string, public filePath: string, public parentFilePath?: string) { + const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '') + const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '') + + super(source, path, parentPath) + } + + async load() { + const contents = await readFile(this.filePath, 'utf8') + + const { checksum, meta, sections } = processMdxForSearch(contents) + + this.checksum = checksum + this.meta = meta + this.sections = sections + + return { + checksum, + meta, + sections, + } + } +} diff --git a/apps/docs/scripts/search/sources/reference-doc.ts b/apps/docs/scripts/search/sources/reference-doc.ts new file mode 100644 index 00000000000..61743a7748f --- /dev/null +++ b/apps/docs/scripts/search/sources/reference-doc.ts @@ -0,0 +1,138 @@ +import { createHash } from 'crypto' +import { readFile } from 'fs/promises' +import yaml from 'js-yaml' +import { OpenAPIV3 } from 'openapi-types' +import { + ICommonFunc, + IFunctionDefinition, + ISpec, +} from '../../../components/reference/Reference.types' +import { CliCommand, CliSpec } from '../../../generator/types/CliSpec' +import { flattenSections } from '../../../lib/helpers' +import { enrichedOperation, gen_v3 } from '../../../lib/refGenerator/helpers' +import { BaseSource, Json } from './base' + +export abstract class ReferenceSource extends BaseSource { + type = 'reference' as const + + constructor( + source: string, + path: string, + public meta: Json, + public specFilePath: string, + public sectionsFilePath: string + ) { + super(source, path) + } + + async load() { + const specContents = await readFile(this.specFilePath, 'utf8') + const refSectionsContents = await readFile(this.sectionsFilePath, 'utf8') + + const refSections: ICommonFunc[] = JSON.parse(refSectionsContents) + const flattenedRefSections = flattenSections(refSections) + + const checksum = createHash('sha256') + .update(specContents + refSectionsContents) + .digest('base64') + + const specSections = this.getSpecSections(specContents) + + const sections = flattenedRefSections + .map((refSection) => { + const specSection = this.matchSpecSection(specSections, refSection.id) + + if (!specSection) { + return + } + + return { + heading: refSection.title, + slug: refSection.slug, + content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection( + specSection, + refSection + )}`, + } + }) + .filter((section) => !!section) + + this.checksum = checksum + this.sections = sections + + return { + checksum, + sections, + meta: this.meta, + } + } + + abstract getSpecSections(specContents: string): SpecSection[] + abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection + abstract formatSection(specSection: SpecSection, refSection: ICommonFunc): string +} + +export class OpenApiReferenceSource extends ReferenceSource { + getSpecSections(specContents: string): enrichedOperation[] { + const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents) + + const generatedSpec = gen_v3(spec, '', { + apiUrl: 'apiv0', + }) + + return generatedSpec.operations + } + matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation { + return operations.find((operation) => operation.operationId === id) + } + formatSection(specOperation: enrichedOperation) { + const { summary, description, operation, path, tags } = specOperation + return JSON.stringify({ + summary, + description, + operation, + path, + tags, + }) + } +} + +export class ClientLibReferenceSource extends ReferenceSource { + getSpecSections(specContents: string): IFunctionDefinition[] { + const spec = yaml.load(specContents) as ISpec + + return spec.functions + } + matchSpecSection(functionDefinitions: IFunctionDefinition[], id: string): IFunctionDefinition { + return functionDefinitions.find((functionDefinition) => functionDefinition.id === id) + } + formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonFunc): string { + const { title } = refSection + const { description, title: functionName } = functionDefinition + + return JSON.stringify({ + title, + description, + functionName, + }) + } +} + +export class CliReferenceSource extends ReferenceSource { + getSpecSections(specContents: string): CliCommand[] { + const spec = yaml.load(specContents) as CliSpec + + return spec.commands + } + matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand { + return cliCommands.find((cliCommand) => cliCommand.id === id) + } + formatSection(cliCommand: CliCommand): string { + const { summary, description, usage } = cliCommand + return JSON.stringify({ + summary, + description, + usage, + }) + } +} diff --git a/apps/docs/scripts/search/sources/util.ts b/apps/docs/scripts/search/sources/util.ts new file mode 100644 index 00000000000..b43bf98cdd1 --- /dev/null +++ b/apps/docs/scripts/search/sources/util.ts @@ -0,0 +1,43 @@ +import { readdir, stat } from 'fs/promises' +import { basename, dirname, join } from 'path' + +export type WalkEntry = { + path: string + parentPath?: string +} + +export async function walk(dir: string, parentPath?: string): Promise { + const immediateFiles = await readdir(dir) + + const recursiveFiles = await Promise.all( + immediateFiles.map(async (file) => { + const path = join(dir, file) + const stats = await stat(path) + if (stats.isDirectory()) { + // Keep track of document hierarchy (if this dir has corresponding doc file) + const docPath = `${basename(path)}.mdx` + + return walk( + path, + immediateFiles.includes(docPath) ? join(dirname(path), docPath) : parentPath + ) + } else if (stats.isFile()) { + return [ + { + path: path, + parentPath, + }, + ] + } else { + return [] + } + }) + ) + + const flattenedFiles = recursiveFiles.reduce( + (all, folderContents) => all.concat(folderContents), + [] + ) + + return flattenedFiles.sort((a, b) => a.path.localeCompare(b.path)) +} diff --git a/apps/www/components/Nav/index.tsx b/apps/www/components/Nav/index.tsx index e178efef081..4f6e5699b1e 100644 --- a/apps/www/components/Nav/index.tsx +++ b/apps/www/components/Nav/index.tsx @@ -2,7 +2,7 @@ import React, { useState } from 'react' import Link from 'next/link' import { useRouter } from 'next/router' -import { Button, Badge, IconStar, IconChevronDown } from 'ui' +import { Button, Badge, Announcement, AnnouncementCountdown, IconStar, IconChevronDown } from 'ui' import FlyOut from '~/components/UI/FlyOut' import Transition from 'lib/Transition' @@ -10,8 +10,7 @@ import SolutionsData from 'data/Solutions.json' import Solutions from '~/components/Nav/Product' import Developers from '~/components/Nav/Developers' -import Announcement from '~/components/Nav/Announcement' -import CountdownBanner from '~/components/LaunchWeek/Banners/CountdownBanner' + import ScrollProgress from '~/components/ScrollProgress' import { useIsLoggedIn, useTheme } from 'common' @@ -19,6 +18,7 @@ import TextLink from '../TextLink' import Image from 'next/image' import * as supabaseLogoWordmarkDark from 'common/assets/images/supabase-logo-wordmark--dark.png' import * as supabaseLogoWordmarkLight from 'common/assets/images/supabase-logo-wordmark--light.png' + import * as supabaseLogoWordmarkWhite from 'common/assets/images/supabase-logo-wordmark--white.png' const Nav = () => { @@ -198,7 +198,7 @@ const Nav = () => { return ( <> - +