+
Sorry, I don't know how to help with that.
+Searching for results
+Sorry, looks like Clippy is having a hard time!
+Please try again in a bit.
+ + tag
// if it's fewer than 70 characters, add a white-space: pre so it doesn't wrap
const shortCodeBlockClasses =
- typeof children === 'string' && children.length < 70 ? 'short-inline-codeblock' : ''
+ typeof codeValue === 'string' && codeValue.length < 70 ? 'short-inline-codeblock' : ''
let lang = language ? language : className ? className.replace('language-', '') : 'js'
// force jsx to be js highlighted
@@ -118,7 +126,7 @@ const CodeBlock: FC = ({
paddingBottom: '4px',
}}
>
- {(value || children)?.trimEnd()}
+ {codeValue}
{!hideCopy && (value || children) && className ? (
= ({
`${!title ? 'top-2' : 'top-[3.25rem]'}`,
].join(' ')}
>
- {/* //
+ {/* //
@ts-ignore */}
export const resources = [
- // removing until the examples page is reworked
- // {
- // title: 'Examples',
- // hasLightIcon: true,
- // href: '/guides/resources/examples',
- // description: 'Official GitHub examples, curated content from the community, and more.',
- // },
+ {
+ title: 'Examples',
+ hasLightIcon: true,
+ href: '/guides/resources/examples',
+ description: 'Official GitHub examples, curated content from the community, and more.',
+ },
{
title: 'Glossary',
hasLightIcon: true,
@@ -111,6 +137,27 @@ export const migrationGuides = [
},
]
+export const postgres = [
+ {
+ title: 'Drop all tables in schema',
+ hasLightIcon: true,
+ href: '/guides/resources/postgres/dropping-all-tables-in-schema',
+ description: 'Delete all tables in a given schema.',
+ },
+ {
+ title: 'Select first row per group',
+ hasLightIcon: true,
+ href: '/guides/resources/postgres/first-row-in-group',
+ description: 'Retrieve the first row in each distinct group.',
+ },
+ {
+ title: 'Print PostgreSQL version',
+ hasLightIcon: true,
+ href: '/guides/resources/postgres/which-version-of-postgres',
+ description: 'Find out which version of Postgres you are running.',
+ },
+]
+
export const Page = ({ children }) =>
+
+This deletes all tables and their associated data. Ensure you have a recent [backup](/docs/guides/platform/backups) before proceeding.
+
+
+
+```sql
+do $$ declare
+ r record;
+begin
+ for r in (select tablename from pg_tables where schemaname = 'my-schema-name') loop
+ execute 'drop table if exists ' || quote_ident(r.tablename) || ' cascade';
+ end loop;
+end $$;
+```
+
+This query works by listing out all the tables in the given schema and then executing a `drop table` for each (hence the `for... loop`).
+
+You can run this query using the [SQL Editor](https://app.supabase.com/project/_/sql) in the Supabase Dashboard, or via `psql` if you're [connecting directly to the database](/docs/guides/database/connecting-to-postgres#direct-connections).
+
+export const Page = ({ children }) =>
+ >((object, property) => {
+ if (property.type !== 'Property') {
+ return object
+ }
+
+ const key = (property.key.type === 'Identifier' && property.key.name) || undefined
+ const value = (property.value.type === 'Literal' && property.value.value) || undefined
+
+ if (!key) {
+ return object
+ }
+
+ return {
+ ...object,
+ [key]: value,
+ }
+ }, {})
+}
+
+/**
+ * Extracts the `meta` ESM export from the MDX file.
+ *
+ * This info is akin to frontmatter.
+ */
+function extractMetaExport(mdxTree: Root) {
+ const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => {
+ return (
+ node.type === 'mdxjsEsm' &&
+ node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
+ node.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
+ node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
+ node.data.estree.body[0].declaration.declarations[0].id.name === 'meta'
+ )
+ })
+
+ if (!metaExportNode) {
+ return undefined
+ }
+
+ const objectExpression =
+ (metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
+ metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
+ metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
+ metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' &&
+ metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type ===
+ 'ObjectExpression' &&
+ metaExportNode.data.estree.body[0].declaration.declarations[0].init) ||
+ undefined
+
+ if (!objectExpression) {
+ return undefined
+ }
+
+ return getObjectFromExpression(objectExpression)
+}
+
+/**
+ * Splits a `mdast` tree into multiple trees based on
+ * a predicate function. Will include the splitting node
+ * at the beginning of each tree.
+ *
+ * Useful to split a markdown file into smaller sections.
+ */
+function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) {
+ return tree.children.reduce((trees, node) => {
+ const [lastTree] = trees.slice(-1)
+
+ if (!lastTree || predicate(node)) {
+ const tree: Root = u('root', [node])
+ return trees.concat(tree)
+ }
+
+ lastTree.children.push(node)
+ return trees
+ }, [])
+}
+
+type Meta = ReturnType
+
+type ProcessedMdx = {
+ checksum: string
+ meta: Meta
+ sections: string[]
+}
+
+/**
+ * Processes MDX content for search indexing.
+ * It extracts metadata, strips it of all JSX,
+ * and splits it into sub-sections based on criteria.
+ */
+function processMdxForSearch(content: string): ProcessedMdx {
+ const checksum = createHash('sha256').update(content).digest('base64')
+
+ const mdxTree = fromMarkdown(content, {
+ extensions: [mdxjs()],
+ mdastExtensions: [mdxFromMarkdown()],
+ })
+
+ const meta = extractMetaExport(mdxTree)
+
+ // Remove all MDX elements from markdown
+ const mdTree = filter(
+ mdxTree,
+ (node) =>
+ ![
+ 'mdxjsEsm',
+ 'mdxJsxFlowElement',
+ 'mdxJsxTextElement',
+ 'mdxFlowExpression',
+ 'mdxTextExpression',
+ ].includes(node.type)
+ )
+
+ if (!mdTree) {
+ return {
+ checksum,
+ meta,
+ sections: [],
+ }
+ }
+
+ const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading')
+
+ const sections = sectionTrees.map((tree) => toMarkdown(tree))
+
+ return {
+ checksum,
+ meta,
+ sections,
+ }
+}
+
+async function walk(dir: string): Promise {
+ const immediateFiles = await readdir(dir)
+
+ const recursiveFiles = await Promise.all(
+ immediateFiles.map(async (file) => {
+ const filePath = join(dir, file)
+ const stats = await stat(filePath)
+ if (stats.isDirectory()) {
+ return walk(filePath)
+ } else if (stats.isFile()) {
+ return [filePath]
+ } else {
+ return []
+ }
+ })
+ )
+
+ const flattenedFiles = recursiveFiles.reduce(
+ (all, folderContents) => all.concat(folderContents),
+ []
+ )
+
+ return flattenedFiles
+}
+
+async function generateEmbeddings() {
+ if (
+ !process.env.NEXT_PUBLIC_SUPABASE_URL ||
+ !process.env.SUPABASE_SERVICE_ROLE_KEY ||
+ !process.env.OPENAI_KEY
+ ) {
+ return console.log(
+ 'Environment variables NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, and OPENAI_KEY are required: skipping embeddings generation'
+ )
+ }
+
+ const supabaseClient = createClient(
+ process.env.NEXT_PUBLIC_SUPABASE_URL,
+ process.env.SUPABASE_SERVICE_ROLE_KEY
+ )
+
+ const markdownFiles = (await walk('pages'))
+ .filter((fileName) => /\.mdx?$/.test(fileName))
+ .filter((fileName) => !ignoredFiles.includes(fileName))
+
+ console.log(`Discovered ${markdownFiles.length} pages`)
+ console.log('Checking which pages are new or have changed')
+
+ for (const markdownFile of markdownFiles) {
+ const path = markdownFile.replace(/^pages/, '').replace(/\.mdx?$/, '')
+ try {
+ const contents = await readFile(markdownFile, 'utf8')
+
+ const { checksum, meta, sections } = processMdxForSearch(contents)
+
+ // Check for existing page in DB and compare checksums
+ const { error: fetchPageError, data: existingPage } = await supabaseClient
+ .from('page')
+ .select()
+ .filter('path', 'eq', path)
+ .limit(1)
+ .maybeSingle()
+
+ if (fetchPageError) {
+ throw fetchPageError
+ }
+
+ // We use checksum to determine if this page & its sections need to be regenerated
+ if (existingPage?.checksum === checksum) {
+ continue
+ }
+
+ console.log({ checksum, path, meta })
+
+ if (existingPage) {
+ console.log(
+ `Docs have changed for '${path}', removing old page sections and their embeddings`
+ )
+
+ const { error: deletePageSectionError } = await supabaseClient
+ .from('page_section')
+ .delete()
+ .filter('page_id', 'eq', existingPage.id)
+
+ if (deletePageSectionError) {
+ throw deletePageSectionError
+ }
+ }
+
+ // Create/update page record. Intentionally clear checksum until we
+ // have successfully generated all page sections.
+ const { error: upsertPageError, data: page } = await supabaseClient
+ .from('page')
+ .upsert({ checksum: null, path, meta }, { onConflict: 'path' })
+ .select()
+ .limit(1)
+ .single()
+
+ if (upsertPageError) {
+ throw upsertPageError
+ }
+
+ console.log(`Adding ${sections.length} page sections (with embeddings) for '${path}'`)
+ for (const section of sections) {
+ // OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
+ const input = section.replace(/\n/g, ' ')
+
+ try {
+ const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY })
+ const openai = new OpenAIApi(configuration)
+
+ const embeddingResponse = await openai.createEmbedding({
+ model: 'text-embedding-ada-002',
+ input,
+ })
+
+ if (embeddingResponse.status !== 200) {
+ throw new Error(inspect(embeddingResponse.data, false, 2))
+ }
+
+ const [responseData] = embeddingResponse.data.data
+
+ const { error: insertPageSectionError, data: pageSection } = await supabaseClient
+ .from('page_section')
+ .insert({
+ page_id: page.id,
+ content: section,
+ token_count: embeddingResponse.data.usage.total_tokens,
+ embedding: responseData.embedding,
+ })
+ .select()
+ .limit(1)
+ .single()
+
+ if (insertPageSectionError) {
+ throw insertPageSectionError
+ }
+ } catch (err) {
+ // TODO: decide how to better handle failed embeddings
+ console.error(
+ `Failed to generate embeddings for '${path}' page section starting with '${input.slice(
+ 0,
+ 40
+ )}...'`
+ )
+
+ throw err
+ }
+ }
+
+ // Set page checksum so that we know this page was stored successfully
+ const { error: updatePageError } = await supabaseClient
+ .from('page')
+ .update({ checksum })
+ .filter('id', 'eq', page.id)
+
+ if (updatePageError) {
+ throw updatePageError
+ }
+ } catch (err) {
+ console.error(
+ `Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.`
+ )
+ console.error(err)
+ }
+ }
+
+ console.log('Embedding generation complete')
+}
+
+async function main() {
+ await generateEmbeddings()
+}
+
+main().catch((err) => console.error(err))
diff --git a/apps/docs/styles/algolia-search.scss b/apps/docs/styles/algolia-search.scss
index 0e2b90589f6..56ac843d5d2 100644
--- a/apps/docs/styles/algolia-search.scss
+++ b/apps/docs/styles/algolia-search.scss
@@ -953,6 +953,7 @@ body {
// search bar
input[type='search'].DocSearch-Input {
background-color: var(--colors-scale1);
+ box-shadow: none;
}
.dark .DocSearch-Input {
diff --git a/apps/docs/types/sse.d.ts b/apps/docs/types/sse.d.ts
new file mode 100644
index 00000000000..dbf97ac0836
--- /dev/null
+++ b/apps/docs/types/sse.d.ts
@@ -0,0 +1,12 @@
+declare module 'sse.js' {
+ export type SSEOptions = EventSourceInit & {
+ headers?: Record
+ payload?: string
+ method?: string
+ }
+
+ export class SSE extends EventSource {
+ constructor(url: string | URL, sseOptions?: SSEOptions)
+ stream(): void
+ }
+}
diff --git a/apps/www/_blog/2020-08-02-supabase-alpha-july-2020.mdx b/apps/www/_blog/2020-08-02-supabase-alpha-july-2020.mdx
index c9042608cc0..6d5847735c1 100644
--- a/apps/www/_blog/2020-08-02-supabase-alpha-july-2020.mdx
+++ b/apps/www/_blog/2020-08-02-supabase-alpha-july-2020.mdx
@@ -41,7 +41,10 @@ We've made some massive improvements to our Table Editor that we're excited to s
Last month we made it easy to drill into your table relationships. This month, we make it possible to drill multiple levels deep.
#### Add, delete, and download rows
@@ -49,7 +52,10 @@ Last month we made it easy to drill into your table relationships. This month, w
We're making it easier to manipulate your data. Next month, you'll be able to add and remove columns directly from the Table view.
### New Postgres Extensions
diff --git a/apps/www/_blog/2020-08-05-supabase-auth.mdx b/apps/www/_blog/2020-08-05-supabase-auth.mdx
index 1b908510843..90b5a7b5e50 100644
--- a/apps/www/_blog/2020-08-05-supabase-auth.mdx
+++ b/apps/www/_blog/2020-08-05-supabase-auth.mdx
@@ -37,7 +37,13 @@ Supabase Auth provides all the backend services you need to authenticate and aut
Supabase makes it simple to onboard your users with our new `supabase.auth.signUp()` and `supabase.auth.signIn()` [functions](/docs/guides/auth).
### Row Level Security
@@ -45,7 +51,13 @@ Supabase makes it simple to onboard your users with our new `supabase.auth.signU
Authentication only gets you so far. When you need granular authorization rules, nothing beats PostgreSQL's [Row Level Security](https://www.postgresql.org/docs/current/ddl-rowsecurity.html). Supabase makes it simple to turn RLS on and off.
### Policies
@@ -53,7 +65,13 @@ Authentication only gets you so far. When you need granular authorization rules,
[Policies](https://www.postgresql.org/docs/current/sql-createpolicy.html) are PostgreSQL's rule engine. They are incredibly powerful and flexible, allowing you to write complex SQL rules which fit your unique business needs.
With policies, your database becomes the rules engine. Instead of repetitively filtering your queries, like this ...
diff --git a/apps/www/_blog/2020-09-03-supabase-alpha-august-2020.mdx b/apps/www/_blog/2020-09-03-supabase-alpha-august-2020.mdx
index ce57ec62933..e20bf8a6a58 100644
--- a/apps/www/_blog/2020-09-03-supabase-alpha-august-2020.mdx
+++ b/apps/www/_blog/2020-09-03-supabase-alpha-august-2020.mdx
@@ -31,7 +31,13 @@ Watch a full demo:
Set up tables and columns directly from the table view.
### Invite your team
@@ -39,7 +45,13 @@ Set up tables and columns directly from the table view.
You can now invite team members to your organisation.
### Auth: Email Confirmations
@@ -47,7 +59,13 @@ You can now invite team members to your organisation.
You can now enable Email Confirmations for new users. This can be toggled on or off and the template for this email can be edited via the dashboard.
### TypeScript support
diff --git a/apps/www/_blog/2020-10-03-supabase-alpha-september-2020.mdx b/apps/www/_blog/2020-10-03-supabase-alpha-september-2020.mdx
index 4a803ba8360..f30c537eb23 100644
--- a/apps/www/_blog/2020-10-03-supabase-alpha-september-2020.mdx
+++ b/apps/www/_blog/2020-10-03-supabase-alpha-september-2020.mdx
@@ -38,7 +38,12 @@ We've released OAuth logins! You can now enable third-party logins on your app f
You can duplicate your tables, just like you would inside a spreadsheet.
### Enable and disable extensions
@@ -46,7 +51,12 @@ You can duplicate your tables, just like you would inside a spreadsheet.
Extensions are easier to use. You can enable Postgres extensions with the click of a button.
### Save your favorite queries
@@ -54,7 +64,12 @@ Extensions are easier to use. You can enable Postgres extensions with the click
The SQL editor now stores your query history in your browser. You can also save your favorite queries to run later!
### GitHub Discussions
diff --git a/apps/www/_blog/2020-12-13-supabase-dashboard-performance.mdx b/apps/www/_blog/2020-12-13-supabase-dashboard-performance.mdx
index af301f4309a..3cfbdb4fcd8 100644
--- a/apps/www/_blog/2020-12-13-supabase-dashboard-performance.mdx
+++ b/apps/www/_blog/2020-12-13-supabase-dashboard-performance.mdx
@@ -74,7 +74,12 @@ Thanks to some skilled haxors (well, weak passwords mainly) we had crypto miners
The spreadsheet import is triggered when the user is creating a new table. However the code was previously loaded every time the page was visited - even when a new table was not being created. This made it a good candidate for lazy loading. Using [Next.js dynamic imports](https://nextjs.org/docs/advanced-features/dynamic-import) we are able to load this component (313 kb brotlied) dynamically, whenever the user clicks the "Add content" button.
We use the same technique to lazy load some Lottie animations which are relatively large.
diff --git a/apps/www/_blog/2021-01-02-supabase-beta-december-2020.mdx b/apps/www/_blog/2021-01-02-supabase-beta-december-2020.mdx
index 7045c3712b9..91bcdeb34ee 100644
--- a/apps/www/_blog/2021-01-02-supabase-beta-december-2020.mdx
+++ b/apps/www/_blog/2021-01-02-supabase-beta-december-2020.mdx
@@ -41,7 +41,12 @@ We spent months working on Performance, Security, and Reliability. Read more on
Add comments and descriptions to your Tables directly from our auto-generated docs. Descriptions are stored as PostgreSQL comments (https://postgresql.org/docs/current/sql-comment.html), and are exposed over your OpenAPI spec.
### Table View now has realtime changes
@@ -49,7 +54,12 @@ Add comments and descriptions to your Tables directly from our auto-generated do
Any updates that happen to your database are reflected in the Table View immediately.
### Table Pagination
@@ -57,7 +67,12 @@ Any updates that happen to your database are reflected in the Table View immedia
Our table view now has pagination - better for working with large data sets.
### Supabase raised a Seed Round
diff --git a/apps/www/_blog/2021-07-27-storage-beta.mdx b/apps/www/_blog/2021-07-27-storage-beta.mdx
index f416eb38feb..a6a5fb30f63 100644
--- a/apps/www/_blog/2021-07-27-storage-beta.mdx
+++ b/apps/www/_blog/2021-07-27-storage-beta.mdx
@@ -21,7 +21,7 @@ Clients can use the `Content-Range` HTTP header to request specific parts of a f
To simplify pricing, we've merged the “Pro” and “Pay as you go” tiers and introduced monthly spend caps to avoid nasty billing surprises.
@@ -136,7 +141,12 @@ We're also retaining our soft limits while we manage the transition to granular
### Database Add-ons
Today we're releasing self-serve Database Add-ons.
@@ -149,7 +159,12 @@ Today, Database Add-ons are available for a small set of customers. We will prog
### New Log Explorer
Today we're releasing a brand new [Log Explorer](/docs/guides/platform/logs) in the Supabase Dashboard.
diff --git a/apps/www/_blog/2022-03-31-supabase-edge-functions.mdx b/apps/www/_blog/2022-03-31-supabase-edge-functions.mdx
index d1446ef35cd..eb07db93572 100644
--- a/apps/www/_blog/2022-03-31-supabase-edge-functions.mdx
+++ b/apps/www/_blog/2022-03-31-supabase-edge-functions.mdx
@@ -81,7 +81,7 @@ We've [extended the Supabase CLI](https://supabase.com/docs/guides/local-develop
Our observability pipeline has been built [using Logflare](https://supabase.com/blog/supabase-acquires-logflare). Structured logs from your Edge Functions get sent to Logflare, which also processes them into metrics. Your Supabase Dashboard is able to reach out to Logflare endpoints, and with some of the [(not so) secret sauce](https://supabase.com/blog/supabase-studio) from our front-end team, you get responsive charts and logs for your Edge Functions.
diff --git a/apps/www/_blog/2022-11-22-flutter-authentication-and-authorization-with-rls.mdx b/apps/www/_blog/2022-11-22-flutter-authentication-and-authorization-with-rls.mdx
index 7183c9fb539..e6ecfbe4cd2 100644
--- a/apps/www/_blog/2022-11-22-flutter-authentication-and-authorization-with-rls.mdx
+++ b/apps/www/_blog/2022-11-22-flutter-authentication-and-authorization-with-rls.mdx
@@ -1351,7 +1351,7 @@ We used bloc for our state management solution. One thing we could have done dif
We could also explore some cool feature improvement.
At the top of the rooms page, we are loading the newest created users to start a conversation. This is fine, but it only allows users to start a conversation with new users.
-We can for example update this to a list of users that are online at the same time. We can implement this using the [presence feature](https://supabase.com/docs/guides/realtime#presence) of Supabase.
+We can for example update this to a list of users that are online at the same time. We can implement this using the [presence feature](https://supabase.com/docs/guides/realtime/presence) of Supabase.
## More Flutter Resources
diff --git a/apps/www/_blog/2023-02-03-openai-embeddings-postgres-vector.mdx b/apps/www/_blog/2023-02-03-openai-embeddings-postgres-vector.mdx
new file mode 100644
index 00000000000..caf2c2bd758
--- /dev/null
+++ b/apps/www/_blog/2023-02-03-openai-embeddings-postgres-vector.mdx
@@ -0,0 +1,377 @@
+---
+title: Storing OpenAI embeddings in Postgres with pgvector
+description: An example of how to build an AI-powered search engine using OpenAI's embeddings and PostgreSQL.
+author: gregnr
+image: embeddings/og_pgvector.png
+thumb: embeddings/og_pgvector.png
+tags:
+ - postgres
+ - ai
+date: '2023-01-06'
+toc_depth: 3
+---
+
+A new PostgreSQL extension is now available in Supabase: [`pgvector`](https://github.com/pgvector/pgvector), an open-source vector similarity search.
+
+The exponential progress of AI functionality over the past year has inspired many new real world applications. One specific challenge has been the ability to store and query _embeddings_ at scale.
+In this post we'll explain what embeddings are, why we might want to use them, and how we can store and query them in PostgreSQL using `pgvector`.
+
+## What are embeddings?
+
+Embeddings capture the “relatedness” of text, images, video, or other types of information. This relatedness is most commonly used for:
+
+- **Search:** how similar is a search term to a body of text?
+- **Recommendations:** how similar are two products?
+- **Classifications:** how do we categorize a body of text?
+- **Clustering:** how do we identify trends?
+
+Let's explore an example of text embeddings. Say we have three phrases:
+
+1. “The cat chases the mouse”
+2. “The kitten hunts rodents”
+3. “I like ham sandwiches”
+
+Your job is to group phrases with similar meaning. If you are a human, this should be obvious. Phrases 1 and 2 are almost identical, while phrase 3 has a completely different meaning.
+
+Although phrases 1 and 2 are similar, they share no common vocabulary (besides “the”). Yet their meanings are nearly identical. How can we teach a computer that these are the same?
+
+## Human language
+
+Humans use words and symbols to communicate language. But words in isolation are mostly meaningless - we need to draw from shared knowledge & experience in order to make sense of them. The phrase “You should Google it” only makes sense if you know that Google is a search engine and that people have been using it as a verb.
+
+In the same way, we need to train a neural network model to understand human language. An effective model should be trained on millions of different examples to understand what each word, phrase, sentence, or paragraph could mean in different contexts.
+
+So how does this relate to embeddings?
+
+## How do embeddings work?
+
+Embeddings compress discrete information (words & symbols) into distributed continuous-valued data (vectors). If we took our phrases from before and plot them on a chart, it might look something like this:
+
+
+
+Phrases 1 and 2 would be plotted close to each other, since their meanings are similar. We would expect phrase 3 to live somewhere far away since it isn't related. If we had a fourth phrase, “Sally ate Swiss cheese”, this might exist somewhere between phrase 3 (cheese can go on sandwiches) and phrase 1 (mice like Swiss cheese).
+
+In this example we only have 2 dimensions: the X and Y axis. In reality, we would need many more dimensions to effectively capture the complexities of human language.
+
+## OpenAI embeddings
+
+OpenAI offers an [API](https://platform.openai.com/docs/guides/embeddings) to generate embeddings for a string of text using its language model. You feed it any text information (blog articles, documentation, your company's knowledge base), and it will output a vector of floating point numbers that represents the “meaning” of that text.
+
+Compared to our 2-dimensional example above, their latest embedding model `text-embedding-ada-002` will output 1536 dimensions.
+
+Why is this useful? Once we have generated embeddings on multiple texts, it is trivial to calculate how similar they are using vector math operations like cosine distance. A perfect use case for this is search. Your process might look something like this:
+
+1. Pre-process your knowledge base and generate embeddings for each page
+2. Store your embeddings to be referenced later (more on this)
+3. Build a search page that prompts your user for input
+4. Take user's input, generate a one-time embedding, then perform a similarity search against your pre-processed embeddings.
+5. Return the most similar pages to the user
+
+## Embeddings in practice
+
+At a small scale, you could store your embeddings in a CSV file, load them into Python, and use a library like `numPy` to calculated similarity between them using something like cosine distance or dot product. OpenAI has a cookbook [example](https://github.com/openai/openai-cookbook/blob/main/examples/Semantic_text_search_using_embeddings.ipynb) that does just that. Unfortunately this likely won't scale well:
+
+- What if I need to store and search over a large number of documents and embeddings (more than can fit in memory)?
+- What if I want to create/update/delete embeddings dynamically?
+- What if I'm not using Python?
+
+### Using PostgreSQL
+
+Enter [`pgvector`](https://github.com/pgvector/pgvector), an extension for PostgreSQL that allows you to both store and query vector embeddings within your database. Let's try it out.
+
+First we'll enable the **Vector** extension. In Supabase, this can be done from the web portal through `Database` → `Extensions`. You can also do this in SQL by running:
+
+```sql
+create extension vector;
+```
+
+Next let's create a table to store our documents and their embeddings:
+
+```sql
+create table documents (
+ id bigserial primary key,
+ content text,
+ embedding vector (1536)
+);
+```
+
+`pgvector` introduces a new data type called `vector`. In the code above, we create a column named `embedding` with the `vector` data type. The size of the vector defines how many dimensions the vector holds. OpenAI's `text-embedding-ada-002` model outputs 1536 dimensions, so we will use that for our vector size.
+
+We also create a `text` column named `content` to store the original document text that produced this embedding. Depending on your use case, you might just store a reference (URL or foreign key) to a document here instead.
+
+Soon we're going to need to perform a similarity search over these embeddings. Let's create a function to do that:
+
+```sql
+create or replace function match_documents (
+ query_embedding vector(1536),
+ similarity_threshold float,
+ match_count int
+)
+returns table (
+ id bigint,
+ content text,
+ similarity float
+)
+language plpgsql
+as $$
+begin
+ return query
+ select
+ id,
+ content,
+ (documents.embedding <=> query_embedding) as similarity
+ from documents
+ where (documents.embedding <=> query_embedding) > similarity_threshold
+ order by documents.embedding <=> query_embedding
+ limit match_count;
+end;
+$$;
+```
+
+`pgvector` introduces 3 new operators that can be used to calculate similarity:
+
+| Operator | Description |
+| -------- | ---------------------- |
+| `<->` | Euclidean distance |
+| `<#>` | negative inner product |
+| `<=>` | cosine distance |
+
+OpenAI recommends cosine similarity on their embeddings, so we will use that here.
+
+Now we can call `match_documents()`, pass in our embedding, similarity threshold, and match count, and we'll get a list of all documents that match. And since this is all managed by Postgres, our application code becomes very simple.
+
+### Indexing
+
+Once your table starts to grow with embeddings, you will likely want to add an index to speed up queries. Vector indexes are particularly important when you're ordering results because vectors are not grouped by similarity, so finding the closest by sequential scan is a resource-intensive operation.
+
+Each distance operator requires a different type of index. We expect to order by cosine distance, so we need `vector_cosine_ops` index. A good starting number of lists is 4 \* sqrt(table_rows):
+
+```sql
+create index on documents
+using ivfflat (embedding vector_cosine_ops)
+with (lists = 100);
+```
+
+You can read more about indexing on `pgvector`'s GitHub page [here](https://github.com/pgvector/pgvector#indexing).
+
+### Generating embeddings
+
+Let's use JavaScript to generate embeddings and store them in Postgres:
+
+```tsx
+import { createClient } from '@supabase/supabase-js'
+import { Configuration, OpenAIApi } from 'openai'
+import { supabaseClient } from './lib/supabase'
+
+async function generateEmbeddings() {
+ const configuration = new Configuration({ apiKey: '' })
+ const openAi = new OpenAIApi(configuration)
+
+ const documents = await getDocuments() // Your custom function to load docs
+
+ // Assuming each document is a string
+ for (const document of documents) {
+ // OpenAI recommends replacing newlines with spaces for best results
+ const input = document.replace(/\n/g, ' ')
+
+ const embeddingResponse = await openai.createEmbedding({
+ model: 'text-embedding-ada-002',
+ input,
+ })
+
+ const [{ embedding }] = embeddingResponse.data.data
+
+ // In production we should handle possible errors
+ await supabaseClient.from('documents').insert({
+ content: document,
+ embedding,
+ })
+ }
+}
+```
+
+### Building a simple search function
+
+Finally, let's create an [Edge Function](https://supabase.com/docs/guides/functions) to perform our similarity search:
+
+```tsx
+import { serve } from 'https://deno.land/std@0.170.0/http/server.ts'
+import 'https://deno.land/x/xhr@0.2.1/mod.ts'
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.5.0'
+import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
+import { supabaseClient } from './lib/supabase'
+
+export const corsHeaders = {
+ 'Access-Control-Allow-Origin': '*',
+ 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+}
+
+serve(async (req) => {
+ // Handle CORS
+ if (req.method === 'OPTIONS') {
+ return new Response('ok', { headers: corsHeaders })
+ }
+
+ // Search query is passed in request payload
+ const { query } = await req.json()
+
+ // OpenAI recommends replacing newlines with spaces for best results
+ const input = query.replace(/\n/g, ' ')
+
+ const configuration = new Configuration({ apiKey: '' })
+ const openai = new OpenAIApi(configuration)
+
+ // Generate a one-time embedding for the query itself
+ const embeddingResponse = await openai.createEmbedding({
+ model: 'text-embedding-ada-002',
+ input,
+ })
+
+ const [{ embedding }] = embeddingResponse.data.data
+
+ // In production we should handle possible errors
+ const { data: documents } = await supabaseClient.rpc('match_documents', {
+ query_embedding: embedding,
+ match_threshold: 0.78, // Choose an appropriate threshold for your data
+ match_count: 10, // Choose the number of matches
+ })
+
+ return new Response(JSON.stringify(documents), {
+ headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+ })
+})
+```
+
+### Building a smarter search function
+
+ChatGPT doesn't just return existing documents. It's able to assimilate a variety of information into a single, cohesive answer. To do this, we need to provide GPT with some relevant documents, and a prompt that it can use to formulate this answer.
+
+One of the biggest challenges of OpenAI's `text-davinci-003` [completion model](https://beta.openai.com/docs/guides/completion) is the 4000 token limit. You must fit both your prompt and the resulting completion within the 4000 tokens. This makes it challenging if you wanted to prompt GPT-3 to answer questions about your own custom knowledge base that would never fit in a single prompt.
+
+Embeddings can help solve this by splitting your prompts into a two-phased process:
+
+1. Query your embedding database for the most relevant documents related to the question
+2. Inject these documents as context for GPT-3 to reference in its answer
+
+Here's another Edge Function that expands upon the simple example above:
+
+```tsx
+import { serve } from 'https://deno.land/std@0.170.0/http/server.ts'
+import 'https://deno.land/x/xhr@0.2.1/mod.ts'
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.5.0'
+import GPT3Tokenizer from 'https://esm.sh/gpt3-tokenizer@1.1.5'
+import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
+import { stripIndent } from 'https://esm.sh/common-tags@1.8.2'
+import { supabaseClient } from './lib/supabase'
+
+export const corsHeaders = {
+ 'Access-Control-Allow-Origin': '*',
+ 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+}
+
+serve(async (req) => {
+ // Handle CORS
+ if (req.method === 'OPTIONS') {
+ return new Response('ok', { headers: corsHeaders })
+ }
+
+ // Search query is passed in request payload
+ const { query } = await req.json()
+
+ // OpenAI recommends replacing newlines with spaces for best results
+ const input = query.replace(/\n/g, ' ')
+
+ const configuration = new Configuration({ apiKey: '' })
+ const openai = new OpenAIApi(configuration)
+
+ // Generate a one-time embedding for the query itself
+ const embeddingResponse = await openai.createEmbedding({
+ model: 'text-embedding-ada-002',
+ input,
+ })
+
+ const [{ embedding }] = embeddingResponse.data.data
+
+ // Fetching whole documents for this simple example.
+ //
+ // Ideally for context injection, documents are chunked into
+ // smaller sections at earlier pre-processing/embedding step.
+ const { data: documents } = await supabaseClient.rpc('match_documents', {
+ query_embedding: embedding,
+ match_threshold: 0.78, // Choose an appropriate threshold for your data
+ match_count: 10, // Choose the number of matches
+ })
+
+ const tokenizer = new GPT3Tokenizer({ type: 'gpt3' })
+ let tokenCount = 0
+ let contextText = ''
+
+ // Concat matched documents
+ for (let i = 0; i < documents.length; i++) {
+ const document = documents[i]
+ const content = document.content
+ const encoded = tokenizer.encode(content)
+ tokenCount += encoded.text.length
+
+ // Limit context to max 1500 tokens (configurable)
+ if (tokenCount > 1500) {
+ break
+ }
+
+ contextText += `${content.trim()}\n---\n`
+ }
+
+ const prompt = stripIndent`${oneLine`
+ You are a very enthusiastic Supabase representative who loves
+ to help people! Given the following sections from the Supabase
+ documentation, answer the question using only that information,
+ outputted in markdown format. If you are unsure and the answer
+ is not explicitly written in the documentation, say
+ "Sorry, I don't know how to help with that."`}
+
+ Context sections:
+ ${contextText}
+
+ Question: """
+ ${query}
+ """
+
+ Answer as markdown (including related code snippets if available):
+ `
+
+ // In production we should handle possible errors
+ const completionResponse = await openai.createCompletion({
+ model: 'text-davinci-003',
+ prompt,
+ max_tokens: 512, // Choose the max allowed tokens in completion
+ temperature: 0, // Set to 0 for deterministic results
+ })
+
+ const {
+ id,
+ choices: [{ text }],
+ } = completionResponse.data
+
+ return new Response(JSON.stringify({ id, text }), {
+ headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+ })
+})
+```
+
+### Streaming results
+
+OpenAI API responses take longer to depending on the length of the “answer”. ChatGPT has a nice UX for this by streaming the response to the user immediately. You can see a similar effect for the Supabase docs:
+
+
+
+The OpenAI API supports [completion streaming](https://platform.openai.com/docs/api-reference/completions/create#completions/create-stream) with Server Side Events. Supabase Edge Functions are run Deno, which also supports [Server Side Events](https://deno.com/blog/deploy-streams#server-sent-events). Check out [this commit](https://github.com/supabase/supabase/pull/12056/commits/bd83e9ba2f7263440888228e3b29007604d94841) to see how we modified the Function above to build a streaming interface.
+
+## Wrap up
+
+Storing embeddings in Postgres opens a world of possibilities. You can combine your search function with telemetry functions, add an user-provided feedback (thumbs up/down), and make your search feel more integrated with your products.
+
+Tomorrow we'll be releasing our own version of this interface in the Supabase docs, to provide Supabase-specific answers.
+
+The pgvector extension is available on all new Supabase projects today. If you want to try it out, launch a new Postgres database today: [database.new](https://database.new)
diff --git a/apps/www/data/home/admin-access.json b/apps/www/data/home/admin-access.json
index 9ca4d408f4e..d6c5f82ad80 100644
--- a/apps/www/data/home/admin-access.json
+++ b/apps/www/data/home/admin-access.json
@@ -11,7 +11,7 @@
"label": "SQL Editor",
"title": "In-built SQL editor for when you need greater control",
"text": "Write, save, and execute SQL queries directly on our dashboard, with templates to save you time. Run common queries and even build applications using our growing list of templates.",
- "video_url": "videos/tabSqlEditor.mp4",
+ "video_url": "https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/marketing/tabSqlEditor.mp4",
"youtube_id": "Hch1mZPZ53A",
"cta": "Explore SQL Editor",
"url": "/database"
@@ -20,7 +20,7 @@
"label": "Auth rules",
"title": "User management as straight-forward as it can be",
"text": "Easily manage your users with Supabase Auth, with email logins, magic links, and third-party logins. Create complex access policies with SQL rules to fit your unique business needs.",
- "video_url": "videos/tabAuthRules.mp4",
+ "video_url": "https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/marketing/tabAuthRules.mp4",
"youtube_id": "vP319FCIZ6Y",
"cta": "Explore Auth",
"url": "/auth"
diff --git a/apps/www/lib/authors.json b/apps/www/lib/authors.json
index 1b2c394d052..584c633d00a 100644
--- a/apps/www/lib/authors.json
+++ b/apps/www/lib/authors.json
@@ -264,5 +264,12 @@
"position": "Engineering",
"author_url": "https://github.com/hf",
"author_image_url": "https://github.com/hf.png"
+ },
+ {
+ "author_id": "gregnr",
+ "author": "Greg Richardson",
+ "position": "Open Source Contributor",
+ "author_url": "https://github.com/gregnr",
+ "author_image_url": "https://github.com/gregnr.png"
}
]
diff --git a/apps/www/lib/redirects.js b/apps/www/lib/redirects.js
index d5fa09975e2..6d2ad98989c 100644
--- a/apps/www/lib/redirects.js
+++ b/apps/www/lib/redirects.js
@@ -1220,13 +1220,13 @@ module.exports = [
},
{
permanent: true,
- source: '/docs/guides/realtime/broadcast',
- destination: '/docs/guides/realtime#broadcast',
+ source: '/docs/guides/realtime#broadcast',
+ destination: '/docs/guides/realtime/broadcast',
},
{
permanent: true,
- source: '/docs/guides/realtime/presence',
- destination: '/docs/guides/realtime#presence',
+ source: '/docs/guides/realtime#presence',
+ destination: '/docs/guides/realtime/presence',
},
{
permanent: true,
@@ -1780,37 +1780,37 @@ module.exports = [
{
permanent: true,
source: '/docs/reference/javascript',
- destination: '/docs/reference/javascript/introduction',
+ destination: '/docs/reference/javascript/start',
},
{
permanent: true,
source: '/docs/reference/dart',
- destination: '/docs/reference/dart/introduction',
+ destination: '/docs/reference/dart/start',
},
{
permanent: true,
source: '/docs/reference/cli',
- destination: '/docs/reference/cli/introduction',
+ destination: '/docs/reference/cli/start',
},
{
permanent: true,
source: '/docs/reference/api',
- destination: '/docs/reference/api/introduction',
+ destination: '/docs/reference/api/start',
},
{
permanent: true,
source: '/docs/reference/auth',
- destination: '/docs/reference/auth/introduction',
+ destination: '/docs/reference/self-hosting-auth/start',
},
{
permanent: true,
source: '/docs/reference/storage',
- destination: '/docs/reference/storage/introduction',
+ destination: '/docs/reference/self-hosting-storage/start',
},
{
permanent: true,
source: '/docs/reference/realtime',
- destination: '/docs/reference/realtime/introduction',
+ destination: '/docs/reference/self-hosting-realtime/start',
},
{
permanent: true,
@@ -1837,4 +1837,9 @@ module.exports = [
source: '/docs/reference/dart/upgrade-guide',
destination: '/docs/reference/dart/v0/upgrade-guide',
},
+ {
+ permanent: true,
+ source: '/docs/guides/examples',
+ destination: '/docs/guides/resources/examples',
+ },
]
diff --git a/apps/www/pages/beta.tsx b/apps/www/pages/beta.tsx
index e4bb383d555..881ad2000fd 100644
--- a/apps/www/pages/beta.tsx
+++ b/apps/www/pages/beta.tsx
@@ -28,7 +28,7 @@ const NavFlyOutMenu = (props: any) => {
return (
<>
-
+
{segments.flat().map((segment: any) => (
{
bg-white px-4 py-4 sm:border-l lg:col-span-4 lg:px-6 lg:py-10`}
>
-
+
0{segment.no}
{segment.name}
@@ -53,12 +53,12 @@ const NavFlyOutMenu = (props: any) => {
const VideoShot = (props: any) => {
const { src } = props
return (
-
+
-
-
-
-
+
+
+
+