mirror of
https://github.com/supabase/supabase.git
synced 2026-10-07 02:15:05 +03:00
Merge branch 'supabase:master' into fix/update-footer-year
This commit is contained in:
21 files changed
+739
-976
No files matched your search
@@ -96,4 +96,6 @@ On the Next.JS side of things, these work almost exactly the same as the client
|
||||
|
||||
#### Search
|
||||
|
||||
Search is handled through Algolia. When the site is built, a [search script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/build-search.ts) runs through all of the types of content, generating search objects that are sent to Algolia to index.
|
||||
Search is handled using a Supabase instance. During CI, [a script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/search/generate-embeddings.ts) aggregates all content sources (eg. guides, reference docs, etc), indexes them using OpenAI embeddings, and stores them in a Supabase database.
|
||||
|
||||
At runtime, an [Edge Function](https://github.com/supabase/supabase/blob/master/supabase/functions) is executed that performs a similarity search between the user's query and the above content sources using [`pgvector`](https://github.com/pgvector/pgvector) embeddings.
|
||||
@@ -8,6 +8,7 @@ import { memo, useEffect } from 'react'
|
||||
import Footer from '~/components/Navigation/Footer'
|
||||
import { menuState, useMenuLevelId, useMenuMobileOpen } from '~/hooks/useMenuState'
|
||||
import Head from 'next/head'
|
||||
import { Announcement, AnnouncementCountdown } from 'ui'
|
||||
|
||||
const levelsData = {
|
||||
home: {
|
||||
@@ -324,6 +325,11 @@ const SiteLayout = ({ children }) => {
|
||||
<title>Supabase Docs</title>
|
||||
</Head>
|
||||
<main>
|
||||
<div>
|
||||
<Announcement>
|
||||
<AnnouncementCountdown />
|
||||
</Announcement>
|
||||
</div>
|
||||
<div className="flex flex-row h-screen">
|
||||
<NavContainer />
|
||||
<Container>
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"build:sitemap": "node ./internals/generate-sitemap.mjs",
|
||||
"embeddings": "tsx scripts/generate-embeddings.ts",
|
||||
"embeddings": "tsx scripts/search/generate-embeddings.ts",
|
||||
"embeddings:refresh": "npm run embeddings -- --refresh",
|
||||
"postbuild": "ts-node ./scripts/build-search.ts && node ./internals/generate-sitemap.mjs",
|
||||
"postbuild": "node ./internals/generate-sitemap.mjs",
|
||||
"generate:all": "npm-run-all --parallel gen:api gen:cli gen:gotrue gen:storage gen:supabase-dart:v0 gen:supabase-dart:v1 gen:supabase-csharp:v0 gen:supabase-js:v1 gen:supabase-js:v2 gen:realtime",
|
||||
"gen:api": "npm-run-all gen:api:usage",
|
||||
"gen:api:usage": "ts-node ./generator/index.ts gen --type api --url https://api.supabase.com --input ../../spec/transforms/api_v0_openapi_deparsed.json --output ./docs/reference/api/generated/usage.mdx",
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
import fs from 'fs'
|
||||
import crypto from 'crypto'
|
||||
import path from 'path'
|
||||
import matter from 'gray-matter'
|
||||
import dotenv from 'dotenv'
|
||||
import algoliasearch from 'algoliasearch/lite'
|
||||
import { isEmpty } from 'lodash'
|
||||
|
||||
// search objects
|
||||
import { generateClientLibSearchObjects } from './files/client-libs'
|
||||
import { generateAPISearchObjects } from './files/api'
|
||||
import { generateCLISearchObjects } from './files/cli'
|
||||
|
||||
const cliObjects = generateCLISearchObjects()
|
||||
const apiObjects = generateAPISearchObjects()
|
||||
const clientLibSearchObjects = generateClientLibSearchObjects()
|
||||
|
||||
// @ts-ignore
|
||||
// The properties of the searchObject that are specifically read
|
||||
// by DocSearch are "type" and "hierarchy". The rest, though saved into Algolia (which we
|
||||
// can potentially use to craft more nuanced search experiences) are not used by DocSearch
|
||||
const ignoredFiles = [
|
||||
'pages/404.mdx',
|
||||
'pages/faq.mdx',
|
||||
'pages/support.mdx',
|
||||
'pages/oss.tsx',
|
||||
'pages/_app.tsx',
|
||||
'pages/_document.tsx',
|
||||
'pages/[...slug].tsx',
|
||||
'pages/handbook/contributing.mdx',
|
||||
'pages/handbook/introduction.mdx',
|
||||
'pages/handbook/supasquad.mdx',
|
||||
]
|
||||
|
||||
async function walk(dir) {
|
||||
let files = await fs.promises.readdir(dir)
|
||||
//@ts-ignore
|
||||
files = await Promise.all(
|
||||
files.map(async (file) => {
|
||||
const filePath = path.join(dir, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) return walk(filePath)
|
||||
else if (stats.isFile()) return filePath
|
||||
})
|
||||
)
|
||||
|
||||
return files.reduce((all, folderContents) => all.concat(folderContents), [])
|
||||
}
|
||||
|
||||
;(async function () {
|
||||
// initialize environment variables
|
||||
dotenv.config()
|
||||
|
||||
if (!process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || !process.env.ALGOLIA_SEARCH_ADMIN_KEY) {
|
||||
return console.log(
|
||||
'Missing Algolia app ID / admin Key: skipping saving of Algolia search index'
|
||||
)
|
||||
}
|
||||
|
||||
console.log('Preparing docs indexing for Algolia')
|
||||
|
||||
try {
|
||||
const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME
|
||||
const client = algoliasearch(
|
||||
process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
|
||||
process.env.ALGOLIA_SEARCH_ADMIN_KEY
|
||||
)
|
||||
const index = client.initIndex(indexName)
|
||||
|
||||
const guidePages = (await walk('pages')).filter((slug) => !ignoredFiles.includes(slug))
|
||||
|
||||
// generate search objects for mdx guide pages
|
||||
const guidePagesearchObjects = guidePages
|
||||
.map((slug) => {
|
||||
let id, title, description
|
||||
const fileContents = fs.readFileSync(slug, 'utf8')
|
||||
const { data, content } = matter(fileContents)
|
||||
|
||||
if (isEmpty(data)) {
|
||||
// Guide pages do not have front-matter meta, unlike reference pages, have to manually extract
|
||||
const metaIndex = fileContents.indexOf('export const meta = {')
|
||||
if (metaIndex !== -1) {
|
||||
const metaString =
|
||||
fileContents
|
||||
.slice(metaIndex + 20, fileContents.indexOf('}', metaIndex + 1) + 1)
|
||||
.replace(/\n/g, '')
|
||||
.slice(0, -2) + '}'
|
||||
const meta = eval(`(${metaString})`)
|
||||
id = meta.id
|
||||
title = meta.title
|
||||
description = meta.description
|
||||
}
|
||||
} else {
|
||||
id = data.id
|
||||
title = data.title
|
||||
description = data.description
|
||||
}
|
||||
|
||||
const url = (slug.includes('/generated') ? slug.replace('/generated', '') : slug)
|
||||
.replace('docs', '')
|
||||
.replace('pages', '')
|
||||
.replace(/\.mdx$/, '')
|
||||
const source = slug.includes('/reference') ? 'reference' : 'guide'
|
||||
|
||||
const object = {
|
||||
// For Algolia
|
||||
objectID: crypto.randomUUID(),
|
||||
id,
|
||||
title,
|
||||
description,
|
||||
url,
|
||||
source,
|
||||
//pageContent: content,
|
||||
pageContent: '',
|
||||
category: undefined,
|
||||
version: undefined,
|
||||
|
||||
// Docsearch specific
|
||||
type: 'lvl1',
|
||||
hierarchy: {
|
||||
lvl0: 'Guides',
|
||||
lvl1: title,
|
||||
lvl2: null,
|
||||
lvl3: null,
|
||||
lvl4: null,
|
||||
lvl5: null,
|
||||
lvl6: null,
|
||||
},
|
||||
}
|
||||
|
||||
return object
|
||||
})
|
||||
// Some of the reference generated files come with an 'index' page that we can ignore
|
||||
.filter((object) => !object.url.endsWith('/index'))
|
||||
.filter((object) => !object.url.endsWith('/.gitkeep'))
|
||||
|
||||
const combinedSearchObjects = guidePagesearchObjects.concat(
|
||||
clientLibSearchObjects,
|
||||
apiObjects,
|
||||
cliObjects
|
||||
)
|
||||
|
||||
//@ts-ignore
|
||||
await index.clearObjects()
|
||||
console.log(`Successfully cleared records from ${indexName}`)
|
||||
|
||||
//@ts-ignore
|
||||
const algoliaResponse = await index.saveObjects(combinedSearchObjects)
|
||||
|
||||
//@ts-ignore
|
||||
console.log(`Successfully saved ${algoliaResponse.objectIDs.length} records into ${indexName}.`)
|
||||
} catch (error) {
|
||||
console.log('Error:', error)
|
||||
}
|
||||
})()
|
||||
@@ -1,43 +0,0 @@
|
||||
import crypto from 'crypto'
|
||||
import { flattenSections } from '../../lib/helpers'
|
||||
|
||||
import apiCommonSections from '~/../../spec/common-api-sections.json'
|
||||
import specFile from '~/../../spec/transforms/api_v0_openapi_deparsed.json'
|
||||
import { gen_v3 } from '../../lib/refGenerator/helpers'
|
||||
|
||||
// @ts-ignore
|
||||
const generatedSpec = gen_v3(specFile, 'wat', { apiUrl: 'apiv0' })
|
||||
const sections = flattenSections(apiCommonSections)
|
||||
|
||||
export function generateAPISearchObjects() {
|
||||
let searchObjects = []
|
||||
|
||||
//@ts-ignore
|
||||
sections.map((section) => {
|
||||
const object = searchObjects.push({
|
||||
objectID: crypto.randomUUID(),
|
||||
id: section.id,
|
||||
title: section.title,
|
||||
// @ts-ignore
|
||||
description: generatedSpec.operations.find((item) => item.operationId === section.id)
|
||||
?.summary,
|
||||
url: `/reference/api/${section.slug}`,
|
||||
source: 'reference',
|
||||
pageContent: '',
|
||||
category: section.product,
|
||||
version: '',
|
||||
type: 'lvl2',
|
||||
hierarchy: {
|
||||
lvl0: 'References',
|
||||
lvl1: 'Management API',
|
||||
lvl2: section.title,
|
||||
lvl3: null,
|
||||
lvl4: null,
|
||||
lvl5: null,
|
||||
lvl6: null,
|
||||
},
|
||||
})
|
||||
return object
|
||||
})
|
||||
return searchObjects
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import crypto from 'crypto'
|
||||
import fs from 'fs'
|
||||
import { flattenSections } from '../../lib/helpers'
|
||||
import cliCommonSections from '~/../../spec/common-cli-sections.json'
|
||||
import yaml from 'js-yaml'
|
||||
|
||||
// @ts-ignore
|
||||
|
||||
const spec = yaml.load(fs.readFileSync(`../../spec/cli_v1_commands.yaml`, 'utf8'))
|
||||
|
||||
const commonSections = flattenSections(cliCommonSections)
|
||||
|
||||
export function generateCLISearchObjects() {
|
||||
let searchObjects = []
|
||||
|
||||
//@ts-ignore
|
||||
spec.commands.map((section) => {
|
||||
const object = searchObjects.push({
|
||||
objectID: crypto.randomUUID(),
|
||||
id: section.id,
|
||||
title: section.title,
|
||||
// @ts-ignore
|
||||
description: section.description?.substr(0, section.description.indexOf('\n')),
|
||||
url: `/reference/cli/${commonSections.find((item) => item.id === section.id)?.slug}`,
|
||||
source: 'reference',
|
||||
pageContent: '',
|
||||
category: section.product,
|
||||
version: '',
|
||||
type: 'lvl2',
|
||||
hierarchy: {
|
||||
lvl0: 'References',
|
||||
lvl1: 'Supabase CLI',
|
||||
lvl2: section.title,
|
||||
lvl3: null,
|
||||
lvl4: null,
|
||||
lvl5: null,
|
||||
lvl6: null,
|
||||
},
|
||||
})
|
||||
return object
|
||||
})
|
||||
return searchObjects
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
import fs from 'fs'
|
||||
import crypto from 'crypto'
|
||||
import yaml from 'js-yaml'
|
||||
import { flattenSections } from '../../lib/helpers'
|
||||
import { nameMap } from '../helpers'
|
||||
|
||||
import commonLibSections from '~/../../spec/common-client-libs-sections.json'
|
||||
|
||||
const clientLibFiles = [
|
||||
{ fileName: 'supabase_js_v2', label: 'javascript', version: 'v2', versionSlug: false },
|
||||
{ fileName: 'supabase_js_v1', label: 'javascript', version: 'v1', versionSlug: true },
|
||||
{ fileName: 'supabase_dart_v1', label: 'dart', version: 'v1', versionSlug: false },
|
||||
{ fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true },
|
||||
{ fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true },
|
||||
]
|
||||
|
||||
const flatCommonLibSections = flattenSections(commonLibSections)
|
||||
|
||||
export function generateClientLibSearchObjects() {
|
||||
// loop through each spec file, find the correspending entry in the common file and grab the title / description / slug
|
||||
let clientLibSearchObjects = []
|
||||
|
||||
clientLibFiles.map((file) => {
|
||||
const specs = yaml.load(fs.readFileSync(`../../spec/${file.fileName}.yml`, 'utf8'))
|
||||
|
||||
//take each function id, find it in the commonLibSections file and return { id, title, slug, description, }
|
||||
//@ts-ignore
|
||||
specs.functions.map((fn) => {
|
||||
const item = flatCommonLibSections.find((section) => section.id === fn.id)
|
||||
if (item) {
|
||||
const object = clientLibSearchObjects.push({
|
||||
objectID: crypto.randomUUID(),
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
description: item.title,
|
||||
url: `/reference/${file.label}/${file.versionSlug ? file.version + '/' : ''}${item.slug}`,
|
||||
source: 'reference',
|
||||
pageContent: '',
|
||||
category: item.product,
|
||||
version: file.version,
|
||||
type: 'lvl2',
|
||||
hierarchy: {
|
||||
lvl0: 'References',
|
||||
lvl1: `${nameMap[file.label]} ${file.version}`,
|
||||
lvl2: item.title,
|
||||
lvl3: file.version,
|
||||
lvl4: null,
|
||||
lvl5: null,
|
||||
lvl6: null,
|
||||
},
|
||||
})
|
||||
return object
|
||||
}
|
||||
})
|
||||
})
|
||||
return clientLibSearchObjects
|
||||
}
|
||||
@@ -1,659 +0,0 @@
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
import { createHash } from 'crypto'
|
||||
import dotenv from 'dotenv'
|
||||
import { ObjectExpression } from 'estree'
|
||||
import { readdir, readFile, stat } from 'fs/promises'
|
||||
import GithubSlugger from 'github-slugger'
|
||||
import yaml from 'js-yaml'
|
||||
import { Content, Root } from 'mdast'
|
||||
import { fromMarkdown } from 'mdast-util-from-markdown'
|
||||
import { mdxFromMarkdown, MdxjsEsm } from 'mdast-util-mdx'
|
||||
import { toMarkdown } from 'mdast-util-to-markdown'
|
||||
import { toString } from 'mdast-util-to-string'
|
||||
import { mdxjs } from 'micromark-extension-mdxjs'
|
||||
import 'openai'
|
||||
import { Configuration, OpenAIApi } from 'openai'
|
||||
import { OpenAPIV3 } from 'openapi-types'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { u } from 'unist-builder'
|
||||
import { filter } from 'unist-util-filter'
|
||||
import { inspect } from 'util'
|
||||
import { ICommonFunc, IFunctionDefinition, ISpec } from '../components/reference/Reference.types'
|
||||
import { CliCommand, CliSpec } from '../generator/types/CliSpec'
|
||||
import { flattenSections } from '../lib/helpers'
|
||||
import { enrichedOperation, gen_v3 } from '../lib/refGenerator/helpers'
|
||||
|
||||
dotenv.config()
|
||||
|
||||
const ignoredFiles = ['pages/404.mdx']
|
||||
|
||||
/**
|
||||
* Extracts ES literals from an `estree` `ObjectExpression`
|
||||
* into a plain JavaScript object.
|
||||
*/
|
||||
function getObjectFromExpression(node: ObjectExpression) {
|
||||
return node.properties.reduce<
|
||||
Record<string, string | number | bigint | true | RegExp | undefined>
|
||||
>((object, property) => {
|
||||
if (property.type !== 'Property') {
|
||||
return object
|
||||
}
|
||||
|
||||
const key = (property.key.type === 'Identifier' && property.key.name) || undefined
|
||||
const value = (property.value.type === 'Literal' && property.value.value) || undefined
|
||||
|
||||
if (!key) {
|
||||
return object
|
||||
}
|
||||
|
||||
return {
|
||||
...object,
|
||||
[key]: value,
|
||||
}
|
||||
}, {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the `meta` ESM export from the MDX file.
|
||||
*
|
||||
* This info is akin to frontmatter.
|
||||
*/
|
||||
function extractMetaExport(mdxTree: Root) {
|
||||
const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => {
|
||||
return (
|
||||
node.type === 'mdxjsEsm' &&
|
||||
node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
|
||||
node.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
|
||||
node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
|
||||
node.data.estree.body[0].declaration.declarations[0].id.name === 'meta'
|
||||
)
|
||||
})
|
||||
|
||||
if (!metaExportNode) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const objectExpression =
|
||||
(metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
|
||||
metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type ===
|
||||
'ObjectExpression' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].init) ||
|
||||
undefined
|
||||
|
||||
if (!objectExpression) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return getObjectFromExpression(objectExpression)
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a `mdast` tree into multiple trees based on
|
||||
* a predicate function. Will include the splitting node
|
||||
* at the beginning of each tree.
|
||||
*
|
||||
* Useful to split a markdown file into smaller sections.
|
||||
*/
|
||||
function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) {
|
||||
return tree.children.reduce<Root[]>((trees, node) => {
|
||||
const [lastTree] = trees.slice(-1)
|
||||
|
||||
if (!lastTree || predicate(node)) {
|
||||
const tree: Root = u('root', [node])
|
||||
return trees.concat(tree)
|
||||
}
|
||||
|
||||
lastTree.children.push(node)
|
||||
return trees
|
||||
}, [])
|
||||
}
|
||||
|
||||
type Meta = ReturnType<typeof extractMetaExport>
|
||||
|
||||
type Section = {
|
||||
content: string
|
||||
heading?: string
|
||||
slug?: string
|
||||
}
|
||||
|
||||
type ProcessedMdx = {
|
||||
checksum: string
|
||||
meta: Meta
|
||||
sections: Section[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes MDX content for search indexing.
|
||||
* It extracts metadata, strips it of all JSX,
|
||||
* and splits it into sub-sections based on criteria.
|
||||
*/
|
||||
function processMdxForSearch(content: string): ProcessedMdx {
|
||||
const checksum = createHash('sha256').update(content).digest('base64')
|
||||
|
||||
const mdxTree = fromMarkdown(content, {
|
||||
extensions: [mdxjs()],
|
||||
mdastExtensions: [mdxFromMarkdown()],
|
||||
})
|
||||
|
||||
const meta = extractMetaExport(mdxTree)
|
||||
|
||||
// Remove all MDX elements from markdown
|
||||
const mdTree = filter(
|
||||
mdxTree,
|
||||
(node) =>
|
||||
![
|
||||
'mdxjsEsm',
|
||||
'mdxJsxFlowElement',
|
||||
'mdxJsxTextElement',
|
||||
'mdxFlowExpression',
|
||||
'mdxTextExpression',
|
||||
].includes(node.type)
|
||||
)
|
||||
|
||||
if (!mdTree) {
|
||||
return {
|
||||
checksum,
|
||||
meta,
|
||||
sections: [],
|
||||
}
|
||||
}
|
||||
|
||||
const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading')
|
||||
|
||||
const slugger = new GithubSlugger()
|
||||
|
||||
const sections = sectionTrees.map((tree) => {
|
||||
const [firstNode] = tree.children
|
||||
|
||||
const heading = firstNode.type === 'heading' ? toString(firstNode) : undefined
|
||||
const slug = heading ? slugger.slug(heading) : undefined
|
||||
|
||||
return {
|
||||
content: toMarkdown(tree),
|
||||
heading,
|
||||
slug,
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
checksum,
|
||||
meta,
|
||||
sections,
|
||||
}
|
||||
}
|
||||
|
||||
type WalkEntry = {
|
||||
path: string
|
||||
parentPath?: string
|
||||
}
|
||||
|
||||
async function walk(dir: string, parentPath?: string): Promise<WalkEntry[]> {
|
||||
const immediateFiles = await readdir(dir)
|
||||
|
||||
const recursiveFiles = await Promise.all(
|
||||
immediateFiles.map(async (file) => {
|
||||
const path = join(dir, file)
|
||||
const stats = await stat(path)
|
||||
if (stats.isDirectory()) {
|
||||
// Keep track of document hierarchy (if this dir has corresponding doc file)
|
||||
const docPath = `${basename(path)}.mdx`
|
||||
|
||||
return walk(
|
||||
path,
|
||||
immediateFiles.includes(docPath) ? join(dirname(path), docPath) : parentPath
|
||||
)
|
||||
} else if (stats.isFile()) {
|
||||
return [
|
||||
{
|
||||
path: path,
|
||||
parentPath,
|
||||
},
|
||||
]
|
||||
} else {
|
||||
return []
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const flattenedFiles = recursiveFiles.reduce(
|
||||
(all, folderContents) => all.concat(folderContents),
|
||||
[]
|
||||
)
|
||||
|
||||
return flattenedFiles.sort((a, b) => a.path.localeCompare(b.path))
|
||||
}
|
||||
|
||||
abstract class BaseEmbeddingSource {
|
||||
checksum?: string
|
||||
meta?: Meta
|
||||
sections?: Section[]
|
||||
|
||||
constructor(public source: string, public path: string, public parentPath?: string) {}
|
||||
|
||||
abstract load(): Promise<{ checksum: string; meta?: Meta; sections: Section[] }>
|
||||
}
|
||||
|
||||
class MarkdownEmbeddingSource extends BaseEmbeddingSource {
|
||||
type: 'markdown' = 'markdown'
|
||||
|
||||
constructor(source: string, public filePath: string, public parentFilePath?: string) {
|
||||
const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '')
|
||||
const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '')
|
||||
|
||||
super(source, path, parentPath)
|
||||
}
|
||||
|
||||
async load() {
|
||||
const contents = await readFile(this.filePath, 'utf8')
|
||||
|
||||
const { checksum, meta, sections } = processMdxForSearch(contents)
|
||||
|
||||
this.checksum = checksum
|
||||
this.meta = meta
|
||||
this.sections = sections
|
||||
|
||||
return {
|
||||
checksum,
|
||||
meta,
|
||||
sections,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
abstract class ReferenceEmbeddingSource<SpecSection> extends BaseEmbeddingSource {
|
||||
type: 'reference' = 'reference'
|
||||
|
||||
constructor(
|
||||
source: string,
|
||||
path: string,
|
||||
public meta: Meta,
|
||||
public specFilePath: string,
|
||||
public sectionsFilePath: string
|
||||
) {
|
||||
super(source, path)
|
||||
}
|
||||
|
||||
async load() {
|
||||
const specContents = await readFile(this.specFilePath, 'utf8')
|
||||
const refSectionsContents = await readFile(this.sectionsFilePath, 'utf8')
|
||||
|
||||
const refSections: ICommonFunc[] = JSON.parse(refSectionsContents)
|
||||
const flattenedRefSections = flattenSections(refSections)
|
||||
|
||||
const checksum = createHash('sha256')
|
||||
.update(specContents + refSectionsContents)
|
||||
.digest('base64')
|
||||
|
||||
const specSections = this.getSpecSections(specContents)
|
||||
|
||||
const sections = flattenedRefSections
|
||||
.map((refSection) => {
|
||||
const specSection = this.matchSpecSection(specSections, refSection.id)
|
||||
|
||||
if (!specSection) {
|
||||
return
|
||||
}
|
||||
|
||||
return {
|
||||
heading: refSection.title,
|
||||
slug: refSection.slug,
|
||||
content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection(
|
||||
specSection,
|
||||
refSection
|
||||
)}`,
|
||||
}
|
||||
})
|
||||
.filter((section) => !!section)
|
||||
|
||||
this.checksum = checksum
|
||||
this.sections = sections
|
||||
|
||||
return {
|
||||
checksum,
|
||||
sections,
|
||||
meta: this.meta,
|
||||
}
|
||||
}
|
||||
|
||||
abstract getSpecSections(specContents: string): SpecSection[]
|
||||
abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection
|
||||
abstract formatSection(specSection: SpecSection, refSection: ICommonFunc): string
|
||||
}
|
||||
|
||||
class OpenApiEmbeddingSource extends ReferenceEmbeddingSource<enrichedOperation> {
|
||||
getSpecSections(specContents: string): enrichedOperation[] {
|
||||
const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents)
|
||||
|
||||
const generatedSpec = gen_v3(spec, '', {
|
||||
apiUrl: 'apiv0',
|
||||
})
|
||||
|
||||
return generatedSpec.operations
|
||||
}
|
||||
matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation {
|
||||
return operations.find((operation) => operation.operationId === id)
|
||||
}
|
||||
formatSection(specOperation: enrichedOperation) {
|
||||
const { summary, description, operation, path, tags } = specOperation
|
||||
return JSON.stringify({
|
||||
summary,
|
||||
description,
|
||||
operation,
|
||||
path,
|
||||
tags,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
class ClientLibEmbeddingSource extends ReferenceEmbeddingSource<IFunctionDefinition> {
|
||||
getSpecSections(specContents: string): IFunctionDefinition[] {
|
||||
const spec = yaml.load(specContents) as ISpec
|
||||
|
||||
return spec.functions
|
||||
}
|
||||
matchSpecSection(functionDefinitions: IFunctionDefinition[], id: string): IFunctionDefinition {
|
||||
return functionDefinitions.find((functionDefinition) => functionDefinition.id === id)
|
||||
}
|
||||
formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonFunc): string {
|
||||
const { title } = refSection
|
||||
const { description, title: functionName } = functionDefinition
|
||||
|
||||
return JSON.stringify({
|
||||
title,
|
||||
description,
|
||||
functionName,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
class CliEmbeddingSource extends ReferenceEmbeddingSource<CliCommand> {
|
||||
getSpecSections(specContents: string): CliCommand[] {
|
||||
const spec = yaml.load(specContents) as CliSpec
|
||||
|
||||
return spec.commands
|
||||
}
|
||||
matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand {
|
||||
return cliCommands.find((cliCommand) => cliCommand.id === id)
|
||||
}
|
||||
formatSection(cliCommand: CliCommand): string {
|
||||
const { summary, description, usage } = cliCommand
|
||||
return JSON.stringify({
|
||||
summary,
|
||||
description,
|
||||
usage,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type EmbeddingSource =
|
||||
| MarkdownEmbeddingSource
|
||||
| OpenApiEmbeddingSource
|
||||
| ClientLibEmbeddingSource
|
||||
| CliEmbeddingSource
|
||||
|
||||
async function generateEmbeddings() {
|
||||
// TODO: use better CLI lib like yargs
|
||||
const args = process.argv.slice(2)
|
||||
const shouldRefresh = args.includes('--refresh')
|
||||
|
||||
if (
|
||||
!process.env.NEXT_PUBLIC_SUPABASE_URL ||
|
||||
!process.env.SUPABASE_SERVICE_ROLE_KEY ||
|
||||
!process.env.OPENAI_KEY
|
||||
) {
|
||||
return console.log(
|
||||
'Environment variables NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, and OPENAI_KEY are required: skipping embeddings generation'
|
||||
)
|
||||
}
|
||||
|
||||
const supabaseClient = createClient(
|
||||
process.env.NEXT_PUBLIC_SUPABASE_URL,
|
||||
process.env.SUPABASE_SERVICE_ROLE_KEY,
|
||||
{
|
||||
auth: {
|
||||
persistSession: false,
|
||||
autoRefreshToken: false,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const embeddingSources: EmbeddingSource[] = [
|
||||
new OpenApiEmbeddingSource(
|
||||
'api',
|
||||
'/reference/api',
|
||||
{ title: 'Management API Reference' },
|
||||
'../../spec/transforms/api_v0_openapi_deparsed.json',
|
||||
'../../spec/common-api-sections.json'
|
||||
),
|
||||
new ClientLibEmbeddingSource(
|
||||
'js-lib',
|
||||
'/reference/javascript',
|
||||
{ title: 'JavaScript Reference' },
|
||||
'../../spec/supabase_js_v2.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
),
|
||||
new ClientLibEmbeddingSource(
|
||||
'dart-lib',
|
||||
'/reference/dart',
|
||||
{ title: 'Dart Reference' },
|
||||
'../../spec/supabase_dart_v1.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
),
|
||||
new ClientLibEmbeddingSource(
|
||||
'python-lib',
|
||||
'/reference/python',
|
||||
{ title: 'Python Reference' },
|
||||
'../../spec/supabase_py_v2.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
),
|
||||
new ClientLibEmbeddingSource(
|
||||
'csharp-lib',
|
||||
'/reference/csharp',
|
||||
{ title: 'C# Reference' },
|
||||
'../../spec/supabase_csharp_v0.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
),
|
||||
new CliEmbeddingSource(
|
||||
'cli',
|
||||
'/reference/cli',
|
||||
{ title: 'CLI Reference' },
|
||||
'../../spec/cli_v1_commands.yaml',
|
||||
'../../spec/common-cli-sections.json'
|
||||
),
|
||||
...(await walk('pages'))
|
||||
.filter(({ path }) => /\.mdx?$/.test(path))
|
||||
.filter(({ path }) => !ignoredFiles.includes(path))
|
||||
.map((entry) => new MarkdownEmbeddingSource('guide', entry.path)),
|
||||
]
|
||||
|
||||
console.log(`Discovered ${embeddingSources.length} pages`)
|
||||
|
||||
if (!shouldRefresh) {
|
||||
console.log('Checking which pages are new or have changed')
|
||||
} else {
|
||||
console.log('Refresh flag set, re-generating all pages')
|
||||
}
|
||||
|
||||
for (const embeddingSource of embeddingSources) {
|
||||
const { type, source, path, parentPath } = embeddingSource
|
||||
|
||||
try {
|
||||
const { checksum, meta, sections } = await embeddingSource.load()
|
||||
|
||||
// Check for existing page in DB and compare checksums
|
||||
const { error: fetchPageError, data: existingPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select('id, path, checksum, parentPage:parent_page_id(id, path)')
|
||||
.filter('path', 'eq', path)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchPageError) {
|
||||
throw fetchPageError
|
||||
}
|
||||
|
||||
type Singular<T> = T extends any[] ? undefined : T
|
||||
|
||||
// We use checksum to determine if this page & its sections need to be regenerated
|
||||
if (!shouldRefresh && existingPage?.checksum === checksum) {
|
||||
const existingParentPage = existingPage?.parentPage as Singular<
|
||||
typeof existingPage.parentPage
|
||||
>
|
||||
|
||||
// If parent page changed, update it
|
||||
if (existingParentPage?.path !== parentPath) {
|
||||
console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`)
|
||||
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select()
|
||||
.filter('path', 'eq', parentPath)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchParentPageError) {
|
||||
throw fetchParentPageError
|
||||
}
|
||||
|
||||
const { error: updatePageError } = await supabaseClient
|
||||
.from('page')
|
||||
.update({ parent_page_id: parentPage?.id })
|
||||
.filter('id', 'eq', existingPage.id)
|
||||
|
||||
if (updatePageError) {
|
||||
throw updatePageError
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (existingPage) {
|
||||
if (!shouldRefresh) {
|
||||
console.log(
|
||||
`[${path}] Docs have changed, removing old page sections and their embeddings`
|
||||
)
|
||||
} else {
|
||||
console.log(`[${path}] Refresh flag set, removing old page sections and their embeddings`)
|
||||
}
|
||||
|
||||
const { error: deletePageSectionError } = await supabaseClient
|
||||
.from('page_section')
|
||||
.delete()
|
||||
.filter('page_id', 'eq', existingPage.id)
|
||||
|
||||
if (deletePageSectionError) {
|
||||
throw deletePageSectionError
|
||||
}
|
||||
}
|
||||
|
||||
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select()
|
||||
.filter('path', 'eq', parentPath)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchParentPageError) {
|
||||
throw fetchParentPageError
|
||||
}
|
||||
|
||||
// Create/update page record. Intentionally clear checksum until we
|
||||
// have successfully generated all page sections.
|
||||
const { error: upsertPageError, data: page } = await supabaseClient
|
||||
.from('page')
|
||||
.upsert(
|
||||
{
|
||||
checksum: null,
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
meta,
|
||||
parent_page_id: parentPage?.id,
|
||||
},
|
||||
{ onConflict: 'path' }
|
||||
)
|
||||
.select()
|
||||
.limit(1)
|
||||
.single()
|
||||
|
||||
if (upsertPageError) {
|
||||
throw upsertPageError
|
||||
}
|
||||
|
||||
console.log(`[${path}] Adding ${sections.length} page sections (with embeddings)`)
|
||||
for (const { slug, heading, content } of sections) {
|
||||
// OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
|
||||
const input = content.replace(/\n/g, ' ')
|
||||
|
||||
try {
|
||||
const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY })
|
||||
const openai = new OpenAIApi(configuration)
|
||||
|
||||
const embeddingResponse = await openai.createEmbedding({
|
||||
model: 'text-embedding-ada-002',
|
||||
input,
|
||||
})
|
||||
|
||||
if (embeddingResponse.status !== 200) {
|
||||
throw new Error(inspect(embeddingResponse.data, false, 2))
|
||||
}
|
||||
|
||||
const [responseData] = embeddingResponse.data.data
|
||||
|
||||
const { error: insertPageSectionError, data: pageSection } = await supabaseClient
|
||||
.from('page_section')
|
||||
.insert({
|
||||
page_id: page.id,
|
||||
slug,
|
||||
heading,
|
||||
content,
|
||||
token_count: embeddingResponse.data.usage.total_tokens,
|
||||
embedding: responseData.embedding,
|
||||
})
|
||||
.select()
|
||||
.limit(1)
|
||||
.single()
|
||||
|
||||
if (insertPageSectionError) {
|
||||
throw insertPageSectionError
|
||||
}
|
||||
} catch (err) {
|
||||
// TODO: decide how to better handle failed embeddings
|
||||
console.error(
|
||||
`Failed to generate embeddings for '${path}' page section starting with '${input.slice(
|
||||
0,
|
||||
40
|
||||
)}...'`
|
||||
)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// Set page checksum so that we know this page was stored successfully
|
||||
const { error: updatePageError } = await supabaseClient
|
||||
.from('page')
|
||||
.update({ checksum })
|
||||
.filter('id', 'eq', page.id)
|
||||
|
||||
if (updatePageError) {
|
||||
throw updatePageError
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.`
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Embedding generation complete')
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await generateEmbeddings()
|
||||
}
|
||||
|
||||
main().catch((err) => console.error(err))
|
||||
@@ -1,9 +0,0 @@
|
||||
export const nameMap = {
|
||||
api: 'Management API',
|
||||
cli: 'Supabase CLI',
|
||||
auth: 'Auth Server',
|
||||
storage: 'Storage Server',
|
||||
postgres: 'Postgres',
|
||||
dart: 'Supabase Flutter Library',
|
||||
javascript: 'Supabase JavaScript Library',
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { createClient } from '@supabase/supabase-js'
|
||||
import dotenv from 'dotenv'
|
||||
import 'openai'
|
||||
import { Configuration, OpenAIApi } from 'openai'
|
||||
import { inspect } from 'util'
|
||||
import { fetchSources } from './sources'
|
||||
|
||||
dotenv.config()
|
||||
|
||||
async function generateEmbeddings() {
|
||||
// TODO: use better CLI lib like yargs
|
||||
const args = process.argv.slice(2)
|
||||
const shouldRefresh = args.includes('--refresh')
|
||||
|
||||
const requiredEnvVars = ['NEXT_PUBLIC_SUPABASE_URL', 'SUPABASE_SERVICE_ROLE_KEY', 'OPENAI_KEY']
|
||||
|
||||
if (requiredEnvVars.some((name) => !process.env[name])) {
|
||||
throw new Error(
|
||||
`Environment variables ${requiredEnvVars.join(
|
||||
', '
|
||||
)} are required: skipping embeddings generation`
|
||||
)
|
||||
}
|
||||
|
||||
const supabaseClient = createClient(
|
||||
process.env.NEXT_PUBLIC_SUPABASE_URL,
|
||||
process.env.SUPABASE_SERVICE_ROLE_KEY,
|
||||
{
|
||||
auth: {
|
||||
persistSession: false,
|
||||
autoRefreshToken: false,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const embeddingSources = await fetchSources()
|
||||
|
||||
console.log(`Discovered ${embeddingSources.length} pages`)
|
||||
|
||||
if (!shouldRefresh) {
|
||||
console.log('Checking which pages are new or have changed')
|
||||
} else {
|
||||
console.log('Refresh flag set, re-generating all pages')
|
||||
}
|
||||
|
||||
for (const embeddingSource of embeddingSources) {
|
||||
const { type, source, path, parentPath } = embeddingSource
|
||||
|
||||
try {
|
||||
const { checksum, meta, sections } = await embeddingSource.load()
|
||||
|
||||
// Check for existing page in DB and compare checksums
|
||||
const { error: fetchPageError, data: existingPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select('id, path, checksum, parentPage:parent_page_id(id, path)')
|
||||
.filter('path', 'eq', path)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchPageError) {
|
||||
throw fetchPageError
|
||||
}
|
||||
|
||||
type Singular<T> = T extends any[] ? undefined : T
|
||||
|
||||
// We use checksum to determine if this page & its sections need to be regenerated
|
||||
if (!shouldRefresh && existingPage?.checksum === checksum) {
|
||||
const existingParentPage = existingPage?.parentPage as Singular<
|
||||
typeof existingPage.parentPage
|
||||
>
|
||||
|
||||
// If parent page changed, update it
|
||||
if (existingParentPage?.path !== parentPath) {
|
||||
console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`)
|
||||
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select()
|
||||
.filter('path', 'eq', parentPath)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchParentPageError) {
|
||||
throw fetchParentPageError
|
||||
}
|
||||
|
||||
const { error: updatePageError } = await supabaseClient
|
||||
.from('page')
|
||||
.update({ parent_page_id: parentPage?.id })
|
||||
.filter('id', 'eq', existingPage.id)
|
||||
|
||||
if (updatePageError) {
|
||||
throw updatePageError
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (existingPage) {
|
||||
if (!shouldRefresh) {
|
||||
console.log(
|
||||
`[${path}] Docs have changed, removing old page sections and their embeddings`
|
||||
)
|
||||
} else {
|
||||
console.log(`[${path}] Refresh flag set, removing old page sections and their embeddings`)
|
||||
}
|
||||
|
||||
const { error: deletePageSectionError } = await supabaseClient
|
||||
.from('page_section')
|
||||
.delete()
|
||||
.filter('page_id', 'eq', existingPage.id)
|
||||
|
||||
if (deletePageSectionError) {
|
||||
throw deletePageSectionError
|
||||
}
|
||||
}
|
||||
|
||||
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
|
||||
.from('page')
|
||||
.select()
|
||||
.filter('path', 'eq', parentPath)
|
||||
.limit(1)
|
||||
.maybeSingle()
|
||||
|
||||
if (fetchParentPageError) {
|
||||
throw fetchParentPageError
|
||||
}
|
||||
|
||||
// Create/update page record. Intentionally clear checksum until we
|
||||
// have successfully generated all page sections.
|
||||
const { error: upsertPageError, data: page } = await supabaseClient
|
||||
.from('page')
|
||||
.upsert(
|
||||
{
|
||||
checksum: null,
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
meta,
|
||||
parent_page_id: parentPage?.id,
|
||||
},
|
||||
{ onConflict: 'path' }
|
||||
)
|
||||
.select()
|
||||
.limit(1)
|
||||
.single()
|
||||
|
||||
if (upsertPageError) {
|
||||
throw upsertPageError
|
||||
}
|
||||
|
||||
console.log(`[${path}] Adding ${sections.length} page sections (with embeddings)`)
|
||||
for (const { slug, heading, content } of sections) {
|
||||
// OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
|
||||
const input = content.replace(/\n/g, ' ')
|
||||
|
||||
try {
|
||||
const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY })
|
||||
const openai = new OpenAIApi(configuration)
|
||||
|
||||
const embeddingResponse = await openai.createEmbedding({
|
||||
model: 'text-embedding-ada-002',
|
||||
input,
|
||||
})
|
||||
|
||||
if (embeddingResponse.status !== 200) {
|
||||
throw new Error(inspect(embeddingResponse.data, false, 2))
|
||||
}
|
||||
|
||||
const [responseData] = embeddingResponse.data.data
|
||||
|
||||
const { error: insertPageSectionError, data: pageSection } = await supabaseClient
|
||||
.from('page_section')
|
||||
.insert({
|
||||
page_id: page.id,
|
||||
slug,
|
||||
heading,
|
||||
content,
|
||||
token_count: embeddingResponse.data.usage.total_tokens,
|
||||
embedding: responseData.embedding,
|
||||
})
|
||||
.select()
|
||||
.limit(1)
|
||||
.single()
|
||||
|
||||
if (insertPageSectionError) {
|
||||
throw insertPageSectionError
|
||||
}
|
||||
} catch (err) {
|
||||
// TODO: decide how to better handle failed embeddings
|
||||
console.error(
|
||||
`Failed to generate embeddings for '${path}' page section starting with '${input.slice(
|
||||
0,
|
||||
40
|
||||
)}...'`
|
||||
)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// Set page checksum so that we know this page was stored successfully
|
||||
const { error: updatePageError } = await supabaseClient
|
||||
.from('page')
|
||||
.update({ checksum })
|
||||
.filter('id', 'eq', page.id)
|
||||
|
||||
if (updatePageError) {
|
||||
throw updatePageError
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.`
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Embedding generation complete')
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await generateEmbeddings()
|
||||
}
|
||||
|
||||
main().catch((err) => console.error(err))
|
||||
@@ -0,0 +1,20 @@
|
||||
export type Json = Record<
|
||||
string,
|
||||
string | number | boolean | null | Json[] | { [key: string]: Json }
|
||||
>
|
||||
|
||||
export type Section = {
|
||||
content: string
|
||||
heading?: string
|
||||
slug?: string
|
||||
}
|
||||
|
||||
export abstract class BaseSource {
|
||||
checksum?: string
|
||||
meta?: Json
|
||||
sections?: Section[]
|
||||
|
||||
constructor(public source: string, public path: string, public parentPath?: string) {}
|
||||
|
||||
abstract load(): Promise<{ checksum: string; meta?: Json; sections: Section[] }>
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { MarkdownSource } from './markdown'
|
||||
import {
|
||||
CliReferenceSource,
|
||||
ClientLibReferenceSource,
|
||||
OpenApiReferenceSource,
|
||||
} from './reference-doc'
|
||||
import { walk } from './util'
|
||||
|
||||
const ignoredFiles = ['pages/404.mdx']
|
||||
|
||||
export type SearchSource =
|
||||
| MarkdownSource
|
||||
| OpenApiReferenceSource
|
||||
| ClientLibReferenceSource
|
||||
| CliReferenceSource
|
||||
|
||||
/**
|
||||
* Fetches all the sources we want to index for search
|
||||
*/
|
||||
export async function fetchSources() {
|
||||
const openApiReferenceSource = new OpenApiReferenceSource(
|
||||
'api',
|
||||
'/reference/api',
|
||||
{ title: 'Management API Reference' },
|
||||
'../../spec/transforms/api_v0_openapi_deparsed.json',
|
||||
'../../spec/common-api-sections.json'
|
||||
)
|
||||
|
||||
const jsLibReferenceSource = new ClientLibReferenceSource(
|
||||
'js-lib',
|
||||
'/reference/javascript',
|
||||
{ title: 'JavaScript Reference' },
|
||||
'../../spec/supabase_js_v2.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
)
|
||||
|
||||
const dartLibReferenceSource = new ClientLibReferenceSource(
|
||||
'dart-lib',
|
||||
'/reference/dart',
|
||||
{ title: 'Dart Reference' },
|
||||
'../../spec/supabase_dart_v1.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
)
|
||||
|
||||
const pythonLibReferenceSource = new ClientLibReferenceSource(
|
||||
'python-lib',
|
||||
'/reference/python',
|
||||
{ title: 'Python Reference' },
|
||||
'../../spec/supabase_py_v2.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
)
|
||||
|
||||
const cSharpLibReferenceSource = new ClientLibReferenceSource(
|
||||
'csharp-lib',
|
||||
'/reference/csharp',
|
||||
{ title: 'C# Reference' },
|
||||
'../../spec/supabase_csharp_v0.yml',
|
||||
'../../spec/common-client-libs-sections.json'
|
||||
)
|
||||
|
||||
const cliReferenceSource = new CliReferenceSource(
|
||||
'cli',
|
||||
'/reference/cli',
|
||||
{ title: 'CLI Reference' },
|
||||
'../../spec/cli_v1_commands.yaml',
|
||||
'../../spec/common-cli-sections.json'
|
||||
)
|
||||
|
||||
const guideSources = (await walk('pages'))
|
||||
.filter(({ path }) => /\.mdx?$/.test(path))
|
||||
.filter(({ path }) => !ignoredFiles.includes(path))
|
||||
.map((entry) => new MarkdownSource('guide', entry.path))
|
||||
|
||||
const sources: SearchSource[] = [
|
||||
openApiReferenceSource,
|
||||
jsLibReferenceSource,
|
||||
dartLibReferenceSource,
|
||||
pythonLibReferenceSource,
|
||||
cSharpLibReferenceSource,
|
||||
cliReferenceSource,
|
||||
...guideSources,
|
||||
]
|
||||
|
||||
return sources
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { createHash } from 'crypto'
|
||||
import { ObjectExpression } from 'estree'
|
||||
import { readFile } from 'fs/promises'
|
||||
import GithubSlugger from 'github-slugger'
|
||||
import { Content, Root } from 'mdast'
|
||||
import { fromMarkdown } from 'mdast-util-from-markdown'
|
||||
import { MdxjsEsm, mdxFromMarkdown } from 'mdast-util-mdx'
|
||||
import { toMarkdown } from 'mdast-util-to-markdown'
|
||||
import { toString } from 'mdast-util-to-string'
|
||||
import { mdxjs } from 'micromark-extension-mdxjs'
|
||||
import { u } from 'unist-builder'
|
||||
import { filter } from 'unist-util-filter'
|
||||
import { BaseSource, Json, Section } from './base'
|
||||
|
||||
/**
|
||||
* Extracts ES literals from an `estree` `ObjectExpression`
|
||||
* into a plain JavaScript object.
|
||||
*/
|
||||
export function getObjectFromExpression(node: ObjectExpression) {
|
||||
return node.properties.reduce<
|
||||
Record<string, string | number | bigint | true | RegExp | undefined>
|
||||
>((object, property) => {
|
||||
if (property.type !== 'Property') {
|
||||
return object
|
||||
}
|
||||
|
||||
const key = (property.key.type === 'Identifier' && property.key.name) || undefined
|
||||
const value = (property.value.type === 'Literal' && property.value.value) || undefined
|
||||
|
||||
if (!key) {
|
||||
return object
|
||||
}
|
||||
|
||||
return {
|
||||
...object,
|
||||
[key]: value,
|
||||
}
|
||||
}, {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the `meta` ESM export from the MDX file.
|
||||
*
|
||||
* This info is akin to frontmatter.
|
||||
*/
|
||||
export function extractMetaExport(mdxTree: Root) {
|
||||
const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => {
|
||||
return (
|
||||
node.type === 'mdxjsEsm' &&
|
||||
node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
|
||||
node.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
|
||||
node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
|
||||
node.data.estree.body[0].declaration.declarations[0].id.name === 'meta'
|
||||
)
|
||||
})
|
||||
|
||||
if (!metaExportNode) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const objectExpression =
|
||||
(metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
|
||||
metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type ===
|
||||
'ObjectExpression' &&
|
||||
metaExportNode.data.estree.body[0].declaration.declarations[0].init) ||
|
||||
undefined
|
||||
|
||||
if (!objectExpression) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return getObjectFromExpression(objectExpression)
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a `mdast` tree into multiple trees based on
|
||||
* a predicate function. Will include the splitting node
|
||||
* at the beginning of each tree.
|
||||
*
|
||||
* Useful to split a markdown file into smaller sections.
|
||||
*/
|
||||
export function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) {
|
||||
return tree.children.reduce<Root[]>((trees, node) => {
|
||||
const [lastTree] = trees.slice(-1)
|
||||
|
||||
if (!lastTree || predicate(node)) {
|
||||
const tree: Root = u('root', [node])
|
||||
return trees.concat(tree)
|
||||
}
|
||||
|
||||
lastTree.children.push(node)
|
||||
return trees
|
||||
}, [])
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes MDX content for search indexing.
|
||||
* It extracts metadata, strips it of all JSX,
|
||||
* and splits it into sub-sections based on criteria.
|
||||
*/
|
||||
export function processMdxForSearch(content: string): ProcessedMdx {
|
||||
const checksum = createHash('sha256').update(content).digest('base64')
|
||||
|
||||
const mdxTree = fromMarkdown(content, {
|
||||
extensions: [mdxjs()],
|
||||
mdastExtensions: [mdxFromMarkdown()],
|
||||
})
|
||||
|
||||
const meta = extractMetaExport(mdxTree)
|
||||
const serializableMeta: Json = JSON.parse(JSON.stringify(meta))
|
||||
|
||||
// Remove all MDX elements from markdown
|
||||
const mdTree = filter(
|
||||
mdxTree,
|
||||
(node) =>
|
||||
![
|
||||
'mdxjsEsm',
|
||||
'mdxJsxFlowElement',
|
||||
'mdxJsxTextElement',
|
||||
'mdxFlowExpression',
|
||||
'mdxTextExpression',
|
||||
].includes(node.type)
|
||||
)
|
||||
|
||||
if (!mdTree) {
|
||||
return {
|
||||
checksum,
|
||||
meta: serializableMeta,
|
||||
sections: [],
|
||||
}
|
||||
}
|
||||
|
||||
const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading')
|
||||
|
||||
const slugger = new GithubSlugger()
|
||||
|
||||
const sections = sectionTrees.map((tree) => {
|
||||
const [firstNode] = tree.children
|
||||
|
||||
const heading = firstNode.type === 'heading' ? toString(firstNode) : undefined
|
||||
const slug = heading ? slugger.slug(heading) : undefined
|
||||
|
||||
return {
|
||||
content: toMarkdown(tree),
|
||||
heading,
|
||||
slug,
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
checksum,
|
||||
meta: serializableMeta,
|
||||
sections,
|
||||
}
|
||||
}
|
||||
|
||||
export type ProcessedMdx = {
|
||||
checksum: string
|
||||
meta: Json
|
||||
sections: Section[]
|
||||
}
|
||||
|
||||
export class MarkdownSource extends BaseSource {
|
||||
type = 'markdown' as const
|
||||
|
||||
constructor(source: string, public filePath: string, public parentFilePath?: string) {
|
||||
const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '')
|
||||
const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '')
|
||||
|
||||
super(source, path, parentPath)
|
||||
}
|
||||
|
||||
async load() {
|
||||
const contents = await readFile(this.filePath, 'utf8')
|
||||
|
||||
const { checksum, meta, sections } = processMdxForSearch(contents)
|
||||
|
||||
this.checksum = checksum
|
||||
this.meta = meta
|
||||
this.sections = sections
|
||||
|
||||
return {
|
||||
checksum,
|
||||
meta,
|
||||
sections,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { createHash } from 'crypto'
|
||||
import { readFile } from 'fs/promises'
|
||||
import yaml from 'js-yaml'
|
||||
import { OpenAPIV3 } from 'openapi-types'
|
||||
import {
|
||||
ICommonFunc,
|
||||
IFunctionDefinition,
|
||||
ISpec,
|
||||
} from '../../../components/reference/Reference.types'
|
||||
import { CliCommand, CliSpec } from '../../../generator/types/CliSpec'
|
||||
import { flattenSections } from '../../../lib/helpers'
|
||||
import { enrichedOperation, gen_v3 } from '../../../lib/refGenerator/helpers'
|
||||
import { BaseSource, Json } from './base'
|
||||
|
||||
export abstract class ReferenceSource<SpecSection> extends BaseSource {
|
||||
type = 'reference' as const
|
||||
|
||||
constructor(
|
||||
source: string,
|
||||
path: string,
|
||||
public meta: Json,
|
||||
public specFilePath: string,
|
||||
public sectionsFilePath: string
|
||||
) {
|
||||
super(source, path)
|
||||
}
|
||||
|
||||
async load() {
|
||||
const specContents = await readFile(this.specFilePath, 'utf8')
|
||||
const refSectionsContents = await readFile(this.sectionsFilePath, 'utf8')
|
||||
|
||||
const refSections: ICommonFunc[] = JSON.parse(refSectionsContents)
|
||||
const flattenedRefSections = flattenSections(refSections)
|
||||
|
||||
const checksum = createHash('sha256')
|
||||
.update(specContents + refSectionsContents)
|
||||
.digest('base64')
|
||||
|
||||
const specSections = this.getSpecSections(specContents)
|
||||
|
||||
const sections = flattenedRefSections
|
||||
.map((refSection) => {
|
||||
const specSection = this.matchSpecSection(specSections, refSection.id)
|
||||
|
||||
if (!specSection) {
|
||||
return
|
||||
}
|
||||
|
||||
return {
|
||||
heading: refSection.title,
|
||||
slug: refSection.slug,
|
||||
content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection(
|
||||
specSection,
|
||||
refSection
|
||||
)}`,
|
||||
}
|
||||
})
|
||||
.filter((section) => !!section)
|
||||
|
||||
this.checksum = checksum
|
||||
this.sections = sections
|
||||
|
||||
return {
|
||||
checksum,
|
||||
sections,
|
||||
meta: this.meta,
|
||||
}
|
||||
}
|
||||
|
||||
abstract getSpecSections(specContents: string): SpecSection[]
|
||||
abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection
|
||||
abstract formatSection(specSection: SpecSection, refSection: ICommonFunc): string
|
||||
}
|
||||
|
||||
export class OpenApiReferenceSource extends ReferenceSource<enrichedOperation> {
|
||||
getSpecSections(specContents: string): enrichedOperation[] {
|
||||
const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents)
|
||||
|
||||
const generatedSpec = gen_v3(spec, '', {
|
||||
apiUrl: 'apiv0',
|
||||
})
|
||||
|
||||
return generatedSpec.operations
|
||||
}
|
||||
matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation {
|
||||
return operations.find((operation) => operation.operationId === id)
|
||||
}
|
||||
formatSection(specOperation: enrichedOperation) {
|
||||
const { summary, description, operation, path, tags } = specOperation
|
||||
return JSON.stringify({
|
||||
summary,
|
||||
description,
|
||||
operation,
|
||||
path,
|
||||
tags,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export class ClientLibReferenceSource extends ReferenceSource<IFunctionDefinition> {
|
||||
getSpecSections(specContents: string): IFunctionDefinition[] {
|
||||
const spec = yaml.load(specContents) as ISpec
|
||||
|
||||
return spec.functions
|
||||
}
|
||||
matchSpecSection(functionDefinitions: IFunctionDefinition[], id: string): IFunctionDefinition {
|
||||
return functionDefinitions.find((functionDefinition) => functionDefinition.id === id)
|
||||
}
|
||||
formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonFunc): string {
|
||||
const { title } = refSection
|
||||
const { description, title: functionName } = functionDefinition
|
||||
|
||||
return JSON.stringify({
|
||||
title,
|
||||
description,
|
||||
functionName,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export class CliReferenceSource extends ReferenceSource<CliCommand> {
|
||||
getSpecSections(specContents: string): CliCommand[] {
|
||||
const spec = yaml.load(specContents) as CliSpec
|
||||
|
||||
return spec.commands
|
||||
}
|
||||
matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand {
|
||||
return cliCommands.find((cliCommand) => cliCommand.id === id)
|
||||
}
|
||||
formatSection(cliCommand: CliCommand): string {
|
||||
const { summary, description, usage } = cliCommand
|
||||
return JSON.stringify({
|
||||
summary,
|
||||
description,
|
||||
usage,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { readdir, stat } from 'fs/promises'
|
||||
import { basename, dirname, join } from 'path'
|
||||
|
||||
export type WalkEntry = {
|
||||
path: string
|
||||
parentPath?: string
|
||||
}
|
||||
|
||||
export async function walk(dir: string, parentPath?: string): Promise<WalkEntry[]> {
|
||||
const immediateFiles = await readdir(dir)
|
||||
|
||||
const recursiveFiles = await Promise.all(
|
||||
immediateFiles.map(async (file) => {
|
||||
const path = join(dir, file)
|
||||
const stats = await stat(path)
|
||||
if (stats.isDirectory()) {
|
||||
// Keep track of document hierarchy (if this dir has corresponding doc file)
|
||||
const docPath = `${basename(path)}.mdx`
|
||||
|
||||
return walk(
|
||||
path,
|
||||
immediateFiles.includes(docPath) ? join(dirname(path), docPath) : parentPath
|
||||
)
|
||||
} else if (stats.isFile()) {
|
||||
return [
|
||||
{
|
||||
path: path,
|
||||
parentPath,
|
||||
},
|
||||
]
|
||||
} else {
|
||||
return []
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const flattenedFiles = recursiveFiles.reduce(
|
||||
(all, folderContents) => all.concat(folderContents),
|
||||
[]
|
||||
)
|
||||
|
||||
return flattenedFiles.sort((a, b) => a.path.localeCompare(b.path))
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import React, { useState } from 'react'
|
||||
import Link from 'next/link'
|
||||
import { useRouter } from 'next/router'
|
||||
|
||||
import { Button, Badge, IconStar, IconChevronDown } from 'ui'
|
||||
import { Button, Badge, Announcement, AnnouncementCountdown, IconStar, IconChevronDown } from 'ui'
|
||||
import FlyOut from '~/components/UI/FlyOut'
|
||||
import Transition from 'lib/Transition'
|
||||
|
||||
@@ -10,8 +10,7 @@ import SolutionsData from 'data/Solutions.json'
|
||||
|
||||
import Solutions from '~/components/Nav/Product'
|
||||
import Developers from '~/components/Nav/Developers'
|
||||
import Announcement from '~/components/Nav/Announcement'
|
||||
import CountdownBanner from '~/components/LaunchWeek/Banners/CountdownBanner'
|
||||
|
||||
import ScrollProgress from '~/components/ScrollProgress'
|
||||
|
||||
import { useIsLoggedIn, useTheme } from 'common'
|
||||
@@ -19,6 +18,7 @@ import TextLink from '../TextLink'
|
||||
import Image from 'next/image'
|
||||
import * as supabaseLogoWordmarkDark from 'common/assets/images/supabase-logo-wordmark--dark.png'
|
||||
import * as supabaseLogoWordmarkLight from 'common/assets/images/supabase-logo-wordmark--light.png'
|
||||
|
||||
import * as supabaseLogoWordmarkWhite from 'common/assets/images/supabase-logo-wordmark--white.png'
|
||||
|
||||
const Nav = () => {
|
||||
@@ -198,7 +198,7 @@ const Nav = () => {
|
||||
return (
|
||||
<>
|
||||
<Announcement>
|
||||
<CountdownBanner />
|
||||
<AnnouncementCountdown />
|
||||
</Announcement>
|
||||
<div className="sticky top-0 z-40 transform" style={{ transform: 'translate3d(0,0,999px)' }}>
|
||||
<div
|
||||
|
||||
@@ -60,6 +60,23 @@ export * from './src/components/Form'
|
||||
// CMD+K
|
||||
export * from './src/components/Command'
|
||||
|
||||
// layout
|
||||
|
||||
// banners
|
||||
export * from './src/layout/banners'
|
||||
|
||||
// config
|
||||
|
||||
// export { default as Config } from './../ui.config'
|
||||
|
||||
// ARCHIVE
|
||||
|
||||
// export * from './src/components/Textarea'
|
||||
|
||||
// AUTH
|
||||
|
||||
// export * from './src/components/Auth'
|
||||
|
||||
// ICONS
|
||||
// export icons
|
||||
export * from './src/components/Icon/icons/IconActivity'
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
import React from 'react'
|
||||
import Link from 'next/link'
|
||||
import Countdown from 'react-countdown'
|
||||
import _announcement from '~/data/Announcement.json'
|
||||
import { AnnouncementProps } from '../../Nav/Announcement'
|
||||
import _announcement from './data/Announcement.json'
|
||||
import { AnnouncementProps } from './Announcement'
|
||||
import { useRouter } from 'next/router'
|
||||
|
||||
interface CountdownStepProps {
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import React, { useEffect, useState } from 'react'
|
||||
|
||||
import _announcement from '~/data/Announcement.json'
|
||||
import _announcement from './data/Announcement.json'
|
||||
import { IconX } from 'ui'
|
||||
import { useRouter } from 'next/router'
|
||||
import { PropsWithChildren } from 'react'
|
||||
File renamed without changes.
@@ -0,0 +1,2 @@
|
||||
export { default as Announcement } from './Announcement'
|
||||
export { default as AnnouncementCountdown } from './Announcement.Countdown'
|
||||
Reference in new issue
Block a user