Merge branch 'master' into create-pull-request/patch

This commit is contained in:
dannykng committed 2023-02-06 15:16:47 -08:00
commit a94a089e5d
44 files changed
+3121 -496

No files matched your search

+2 -1
View File
@@ -3,4 +3,5 @@
node_modules
package-lock.json
docker*
apps/**/out
apps/**/out
**/**.mdx
+8
View File
@@ -0,0 +1,8 @@
# PUBLIC
NEXT_PUBLIC_NEW_DOCS=true
NEXT_PUBLIC_SUPABASE_URL=http://localhost:54321
NEXT_PUBLIC_SUPABASE_ANON_KEY=
# PRIVATE
OPENAI_KEY=
SUPABASE_SERVICE_ROLE_KEY=
+35
View File
@@ -0,0 +1,35 @@
import { useTheme } from 'common/Providers'
import Image from 'next/image'
import { FC } from 'react'
import { createPortal } from 'react-dom'
import clippyImageDark from '../../public/img/clippy-dark.png'
import clippyImage from '../../public/img/clippy.png'
import ClippyBubble from './ClippyBubble'
import ClippyModal from './ClippyModal'
import { useClippy } from './ClippyProvider'
import { useFlag } from '~/hooks/useFlag'
const Clippy: FC = () => {
const { isDarkMode } = useTheme()
const { isOpen, open, close } = useClippy()
const enableClippy = useFlag('enableClippy')
return (
<>
{!isOpen && (
<div className="hidden md:flex flex-col items-end gap-1 md:gap-4">
{enableClippy && (
<>
<ClippyBubble onClick={open} />
<div className="w-[80px] lg:w-[120px] p-8 md:p-0">
<Image src={isDarkMode ? clippyImageDark : clippyImage} alt="Clippy" />
</div>
</>
)}
</div>
)}
{isOpen && createPortal(<ClippyModal onClose={close} />, document.body)}
</>
)
}
export default Clippy
@@ -0,0 +1,36 @@
import { FC } from 'react'
import { IconCommand } from 'ui'
import Image from 'next/image'
interface Props {
className?: string
onClick?: () => void
}
const ClippyBubble: FC<Props> = ({ className, onClick }) => {
return (
<div
className={`flex gap-2 border bg-white dark:bg-scale-300 relative rounded-xl p-4 lg:p-6 max-w-sm lg:max-w-xs shadow-xl mb-1 lg:mb-4 cursor-pointer hover:bg-scale-200 transition-colors border-scale-500 hover:border-scale-700 overflow-hidden ${
className ? className : ''
}`}
onClick={onClick}
>
<div className="z-0 absolute -left-24 -top-4 w-[260px]">
<Image alt="Clippy blur" width={260} height={260} src="/docs/img/purple-glow-dark.png" />
</div>
What can I help you with?
<div className="flex items-center space-x-1">
<div className="text-scale-1200 flex items-center justify-center h-5 w-10 border rounded bg-scale-500 border-scale-700 gap-1">
<IconCommand size={12} strokeWidth={1.5} />
<span className="text-[12px]">/</span>
</div>
</div>
<div className="absolute -bottom-4 right-[90px] text-scale-600">
<svg viewBox="0 0 1 1" width={16} fill="currentColor">
<path d="M0,0 L1,0 L1,1 Z" />
</svg>
</div>
</div>
)
}
export default ClippyBubble
+267
View File
@@ -0,0 +1,267 @@
import type { CreateCompletionResponse } from 'openai'
import { FC, useCallback, useState } from 'react'
import ReactMarkdown from 'react-markdown'
import remarkGfm from 'remark-gfm'
import { SSE } from 'sse.js'
import clippyImageDark from '../../public/img/clippy-dark.png'
import clippyImage from '../../public/img/clippy.png'
import { useTheme } from 'common/Providers'
import Image from 'next/image'
import {
Button,
IconAlertCircle,
IconAlertTriangle,
IconLoader,
IconSearch,
Input,
Loading,
Modal,
} from 'ui'
import components from '~/components'
type Props = {
onClose?: () => void
}
const questions = [
'How do I get started with Supabase?',
'How do I run Supabase locally?',
'How do I connect to my database?',
'How do I run migrations? ',
'How do I listen to changes in a table?',
'How do I setup authentication?',
]
function getEdgeFunctionUrl() {
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL.replace(/\/$/, '')
const isPlatform = supabaseUrl.match(/(supabase\.co)|(supabase\.in)/)
if (isPlatform) {
const [schemeAndProjectId, domain, tld] = supabaseUrl.split('.')
return `${schemeAndProjectId}.functions.${domain}.${tld}`
} else {
return `${supabaseUrl}/functions/v1`
}
}
const edgeFunctionUrl = getEdgeFunctionUrl()
const ClippyModal: FC<Props> = ({ onClose }) => {
const { isDarkMode } = useTheme()
const [query, setQuery] = useState('')
const [answer, setAnswer] = useState('')
const [isLoading, setIsLoading] = useState(false)
const [isResponding, setIsResponding] = useState(false)
const [hasError, setHasError] = useState(false)
const cantHelp = answer?.trim() === "Sorry, I don't know how to help with that."
const status = isLoading
? 'Clippy is searching...'
: isResponding
? 'Clippy is responding...'
: cantHelp || hasError
? 'Clippy has failed you'
: undefined
const handleConfirm = useCallback(async (query: string) => {
setHasError(false)
setAnswer(undefined)
setIsLoading(true)
const eventSource = new SSE(`${edgeFunctionUrl}/clippy-search`, {
headers: {
apikey: process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY,
Authorization: `Bearer ${process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY}`,
'Content-Type': 'application/json',
},
payload: JSON.stringify({ query }),
})
function handleError<T>(err: T) {
setIsLoading(false)
setIsResponding(false)
setHasError(true)
console.error(err)
}
eventSource.addEventListener('error', handleError)
eventSource.addEventListener('message', (e) => {
try {
setIsLoading(false)
if (e.data === '[DONE]') {
setIsResponding(false)
return
}
setIsResponding(true)
const completionResponse: CreateCompletionResponse = JSON.parse(e.data)
const [{ text }] = completionResponse.choices
setAnswer((answer) => {
return (answer ?? '') + text
})
} catch (err) {
handleError(err)
}
})
eventSource.stream()
setIsLoading(true)
}, [])
function handleResetPrompt() {
setQuery('')
setAnswer(undefined)
setIsResponding(false)
setHasError(false)
}
return (
<Modal size="xlarge" visible={true} onCancel={onClose} closable={false} hideFooter>
<div
className={`mx-auto max-h-[75vh] flex flex-col gap-4 rounded-lg p-4 md:pt-6 md:px-6 pb-2 w-full shadow-2xl overflow-hidden border text-left border-scale-500 bg-scale-100 dark:bg-scale-300 cursor-auto relative min-w-[340px]`}
onClick={(e) => e.stopPropagation()}
>
<div className="relative">
<Input
className="w-full"
size="xlarge"
autoFocus
placeholder="Ask me anything about Supabase"
value={query}
onChange={(e) => setQuery(e.target.value)}
icon={<IconSearch size="small" />}
onKeyDown={(e) => {
switch (e.key) {
case 'Enter':
handleConfirm(query)
return
default:
return
}
}}
/>
<div className="absolute right-0 top-0 mt-3 mr-4 hidden md:block">
<Button type="default" size="tiny" onClick={onClose}>
esc
</Button>
</div>
{!isLoading && answer && (
<div className="absolute right-0 top-0 mt-3 mr-16 hidden md:block">
<Button type="text" size="tiny" onClick={handleResetPrompt}>
Try again
</Button>
</div>
)}
</div>
{!isLoading && !answer && !hasError && (
<div className="">
<div className="mt-2">
<h2 className="text-sm text-scale-1100">Not sure where to start?</h2>
<ul className="text-sm mt-4 text-scale-1100 grid md:flex gap-4 flex-wrap max-w-3xl">
{questions.map((question) => (
<li>
<button
className="hover:bg-slate-400 hover:dark:bg-slate-400 px-4 py-2 bg-slate-300 dark:bg-slate-200 rounded-lg transition-colors"
onClick={() => {
setQuery(question)
handleConfirm(question)
}}
>
{question}
</button>
</li>
))}
</ul>
</div>
</div>
)}
{answer && (
<div className="px-4 py-4 rounded-lg overflow-y-auto bg-scale-200">
{cantHelp ? (
<p className="flex flex-col gap-4 items-center p-4">
<div className="grid md:flex items-center gap-2 mt-4 text-center justify-items-center">
<IconAlertCircle />
<p>Sorry, I don&apos;t know how to help with that.</p>
</div>
<Button size="tiny" type="secondary" onClick={handleResetPrompt}>
Try again?
</Button>
</p>
) : (
<div className="prose dark:prose-dark">
<ReactMarkdown
linkTarget="_blank"
remarkPlugins={[remarkGfm]}
transformLinkUri={(href) => {
const supabaseUrl = new URL('https://supabase.com')
const linkUrl = new URL(href, 'https://supabase.com')
if (linkUrl.origin === supabaseUrl.origin) {
return linkUrl.toString()
}
return href
}}
components={components}
>
{answer}
</ReactMarkdown>
</div>
)}
</div>
)}
{isLoading && (
<div className="p-6 grid gap-6 mt-4">
<Loading active>{}</Loading>
<p className="text-lg text-center">Searching for results</p>
</div>
)}
{hasError && (
<div className="p-6 flex flex-col items-center gap-6 mt-4">
<IconAlertTriangle strokeWidth={1.5} size={40} />
<p className="text-lg text-center">Sorry, looks like Clippy is having a hard time!</p>
<p className="text-sm text-center">Please try again in a bit.</p>
<Button size="tiny" type="secondary" onClick={handleResetPrompt}>
Try again?
</Button>
</div>
)}
<div className="border-t border-scale-600 mt-4 text-scale-1100">
<div className="flex justify-between items-center py-2 text-xs">
<div className="flex items-centerp gap-1 pt-3 pb-1">
<span>Powered by OpenAI.</span>
<a href="/blog/clippy" className="underline">
Read the blog post
</a>
</div>
<div className="flex items-center gap-6 py-1">
{status ? (
<span className="bg-scale-400 rounded-lg py-1 px-2 items-center gap-2 hidden md:flex">
{(isLoading || isResponding) && <IconLoader size={14} className="animate-spin" />}
{status}
</span>
) : (
<></>
)}
<Image
width={30}
height={34}
src={isDarkMode ? clippyImageDark : clippyImage}
alt="Clippy"
/>
</div>
</div>
</div>
</div>
</Modal>
)
}
export default ClippyModal
@@ -0,0 +1,72 @@
import {
createContext,
FC,
PropsWithChildren,
useCallback,
useContext,
useEffect,
useState,
} from 'react'
export type ClippyContextValue = {
isOpen: boolean
open: () => void
close: () => void
}
export const ClippyContext = createContext<ClippyContextValue>(null)
export const useClippy = () => {
const { isOpen, open, close } = useContext(ClippyContext)
return { isOpen, open, close }
}
const ClippyProvider: FC<PropsWithChildren<{}>> = ({ children }) => {
const [isModalOpen, setIsModalOpen] = useState(false)
const onOpen = useCallback(() => {
setIsModalOpen(true)
document.body.classList.add('DocSearch--active')
}, [])
const onClose = useCallback(() => {
setIsModalOpen(false)
document.body.classList.remove('DocSearch--active')
}, [])
useSearchKeyboardEvents({
onOpen,
onClose,
})
return (
<ClippyContext.Provider value={{ isOpen: isModalOpen, open: onOpen, close: onClose }}>
{children}
</ClippyContext.Provider>
)
}
function useSearchKeyboardEvents({ onOpen, onClose }) {
useEffect(() => {
function onKeyDown(event: KeyboardEvent) {
switch (event.key) {
case 'Escape':
onClose()
return
case '/':
if (event.metaKey || event.ctrlKey) {
onOpen()
}
return
}
}
window.addEventListener('keydown', onKeyDown)
return () => {
window.removeEventListener('keydown', onKeyDown)
}
}, [onOpen, onClose])
}
export default ClippyProvider
+12 -4
View File
@@ -1,4 +1,4 @@
import { FC } from 'react'
import { Children, FC } from 'react'
import CopyToClipboard from 'react-copy-to-clipboard'
import { Light as SyntaxHighlighter } from 'react-syntax-highlighter'
import monokaiCustomTheme from './CodeBlock.utils'
@@ -49,10 +49,18 @@ const CodeBlock: FC<Props> = ({
}, 1000)
}
// Extract string when `children` has a single string node
const childrenArray = Children.toArray(children)
const [singleChild] = childrenArray.length === 1 ? childrenArray : []
const singleString = typeof singleChild === 'string' ? singleChild : undefined
let codeValue = value ?? singleString ?? children
codeValue = codeValue?.trimEnd?.() ?? codeValue
// check the length of the string inside the <code> tag
// if it's fewer than 70 characters, add a white-space: pre so it doesn't wrap
const shortCodeBlockClasses =
typeof children === 'string' && children.length < 70 ? 'short-inline-codeblock' : ''
typeof codeValue === 'string' && codeValue.length < 70 ? 'short-inline-codeblock' : ''
let lang = language ? language : className ? className.replace('language-', '') : 'js'
// force jsx to be js highlighted
@@ -118,7 +126,7 @@ const CodeBlock: FC<Props> = ({
paddingBottom: '4px',
}}
>
{(value || children)?.trimEnd()}
{codeValue}
</SyntaxHighlighter>
{!hideCopy && (value || children) && className ? (
<div
@@ -128,7 +136,7 @@ const CodeBlock: FC<Props> = ({
`${!title ? 'top-2' : 'top-[3.25rem]'}`,
].join(' ')}
>
{/* //
{/* //
@ts-ignore */}
<CopyToClipboard text={value || children}>
<Button
+17
View File
@@ -0,0 +1,17 @@
import React, { ReactNode } from 'react'
import { useFlag } from '~/hooks/useFlag'
interface Props {
children: ReactNode
name: string
}
function Flag({ children, name }: Props) {
const flagValue = useFlag(name)
if (!flagValue) {
return null
}
return <>{children}</>
}
export default Flag
+5
View File
@@ -0,0 +1,5 @@
import { createContext } from 'react'
const FlagContext = createContext({})
export default FlagContext
@@ -0,0 +1,36 @@
import { FC, useEffect, useState } from 'react'
import createConfigCatClient from 'configcat-js'
import FlagContext from './FlagContext'
const FlagProvider: FC = ({ children }) => {
const [store, setStore] = useState({})
const { Provider } = FlagContext
useEffect(() => {
getFlags()
}, [])
const getFlags = async () => {
const setFlagValues = async () => {
const flagValues = await client.getAllValuesAsync()
const flagStore: any = {}
flagValues.forEach((item: any) => {
flagStore[item.settingKey] = item.settingValue
})
setStore(flagStore)
}
const client = createConfigCatClient(process.env.NEXT_PUBLIC_CONFIGCAT_SDK_KEY ?? '', {
configChanged: setFlagValues,
pollIntervalSeconds: 600,
})
await setFlagValues()
}
return <Provider value={store}>{children}</Provider>
}
export default FlagProvider
@@ -419,6 +419,11 @@ export const database = {
url: '/guides/database/extensions/pgnet',
items: [],
},
{
name: 'pgvector: Embeddings and vector similarity',
url: '/guides/database/extensions/pgvector',
items: [],
},
{
name: 'pg_stat_statements: SQL Planning and Execution Statistics',
url: '/guides/database/extensions/pg_stat_statements',
@@ -812,7 +817,7 @@ export const reference = {
url: '/reference/python/start',
level: 'reference_python',
items: [],
icon: 'docs/img/icons/javascript.svg',
icon: '/img/icons/menu/reference-python',
},
// {
// name: 'supabase-python',
@@ -6,13 +6,17 @@ import { Button, IconCommand, IconGitHub, IconMoon, IconSearch, IconSun } from '
import { SearchButton } from '~/components/DocSearch'
import { REFERENCES } from '~/components/Navigation/Navigation.constants'
import { useTheme } from 'common/Providers'
import clippyImageDark from '../../../public/img/clippy-dark.png'
import clippyImage from '../../../public/img/clippy.png'
import { getPageType } from '~/lib/helpers'
import { useClippy } from '~/components/Clippy/ClippyProvider'
const TopNavBarRef: FC = () => {
const { isDarkMode, toggleTheme } = useTheme()
const [mounted, setMounted] = useState(false)
const [mobileMenuOpen, setMobileMenuOpen] = useState(false)
const { open: openClippy } = useClippy()
const { asPath, push } = useRouter()
const pathSegments = asPath.split('/')
@@ -84,8 +88,17 @@ const TopNavBarRef: FC = () => {
{/* {router.asPath.includes('/reference/') && <RefSwitcher />} */}
</div>
<div className="flex items-center gap-12">
<SearchButton className="w-full lg:w-96">
<div className="flex items-center gap-6">
<div className="flex md:hidden cursor-pointer">
<Image
onClick={openClippy}
width={26}
height={29}
src={isDarkMode ? clippyImageDark : clippyImage}
alt="Clippy"
/>
</div>
<SearchButton className="md:w-full lg:w-96">
<div
className="
flex
-122
View File
@@ -1,122 +0,0 @@
---
id: initializing
title: 'Initializing'
slug: initializing
custom_edit_url: https://github.com/supabase/supabase/edit/master/web/spec/supabase.yml
---
### Initializing a new Client
<RefSubLayout.EducationRow>
<RefSubLayout.Details>
Initializing a new client is pretty straightforward. Find your project url and public key from the
admin panel and pass it into your client initialization function.
`supabase-csharp` is heavily dependent on Models deriving from `BaseModel`. To interact with the API, one must have the associated model (see example) specified.
Leverage `Table`, `PrimaryKey`, and `Column` attributes to specify names of classes/properties that are different from their C# Versions.
</RefSubLayout.Details>
<RefSubLayout.Examples>
<Tabs
size="small"
type="underlined"
defaultActiveId="csharp-init-standard">
<TabPanel id="csharp-init-standard" label="Standard">
```csharp
var url = Environment.GetEnvironmentVariable("SUPABASE_URL");
var key = Environment.GetEnvironmentVariable("SUPABASE_KEY");
var options = new Supabase.SupabaseOptions
{
AutoConnectRealtime = true
};
var supabase = new Supabase.Client(url, key, options);
await supabase.InitializeAsync();
```
</TabPanel>
<TabPanel id="csharp-init-maui" label="Dependency Injection (Maui-like)">
```csharp
public static MauiApp CreateMauiApp()
{
// ...
var builder = MauiApp.CreateBuilder();
var url = Environment.GetEnvironmentVariable("SUPABASE_URL");
var key = Environment.GetEnvironmentVariable("SUPABASE_KEY");
var options = new SupabaseOptions
{
AutoRefreshToken = true,
AutoConnectRealtime = true,
SessionHandler = new SupabaseSessionHandler()
};
// Note the creation as a singleton.
builder.Services.AddSingleton(provider => new Supabase.Client(url, key, options));
}
```
</TabPanel>
<TabPanel id="csharp-init-showing-models" label="With Models Example">
```csharp
// Given the following Model representing the Supabase Database (Message.cs)
[Table("messages")]
public class Message : BaseModel
{
[PrimaryKey("id")]
public int Id { get; set; }
[Column("username")]
public string UserName { get; set; }
[Column("channel_id")]
public int ChannelId { get; set; }
public override bool Equals(object obj)
{
return obj is Message message &&
Id == message.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
}
void Initialize()
{
// Get All Messages
var response = await client.Table<Message>().Get();
List<Message> models = response.Models;
// Insert
var newMessage = new Message { UserName = "acupofjose", ChannelId = 1 };
await client.Table<Message>().Insert();
// Update
var model = response.Models.First();
model.UserName = "elrhomariyounes";
await model.Update();
// Delete
await response.Models.Last().Delete();
// etc.
}
```
</TabPanel>
</Tabs>
</RefSubLayout.Examples>
</RefSubLayout.EducationRow>
+7
View File
@@ -0,0 +1,7 @@
import { useContext } from 'react'
import FlagContext from 'components/Flag/FlagContext'
export function useFlag(name: string) {
const store: any = useContext(FlagContext)
return store[name]
}
+4
View File
@@ -7,6 +7,7 @@ import TopNavBarRef from '~/components/Navigation/NavigationMenu/TopNavBarRef'
import FooterHelpCallout from '~/components/FooterHelpCallout'
import { memo, useEffect } from 'react'
import Clippy from '~/components/Clippy/Clippy'
import Footer from '~/components/Navigation/Footer'
import { menuState, useMenuLevelId, useMenuMobileOpen } from '~/hooks/useMenuState'
@@ -338,6 +339,9 @@ const SiteLayout = ({ children }) => {
</div>
<MobileMenuBackdrop />
</Container>
<div className=" absolute right-4 md:right-12 bottom-8 z-[300]">
<Clippy />
</div>
</div>
</main>
)
+1 -1
View File
@@ -84,6 +84,6 @@ const nextConfig = {
// next.config.js
export default () => {
// const plugins = [withMDX]/
const plugins = [withTM(['ui', 'common']), withMDX, withYaml]
const plugins = [withTM(['ui', 'common', '@supabase/auth-helpers-nextjs']), withMDX, withYaml]
return plugins.reduce((acc, next) => next(acc), nextConfig)
}
+14
View File
@@ -8,6 +8,7 @@
"start": "next start",
"lint": "next lint",
"build:sitemap": "node ./internals/generate-sitemap.mjs",
"build:embeddings": "tsx scripts/generate-embeddings.ts",
"postbuild": "ts-node ./scripts/build-search.ts && node ./internals/generate-sitemap.mjs",
"generate:all": "npm-run-all --parallel gen:api gen:cli gen:gotrue gen:storage gen:supabase-dart:v0 gen:supabase-dart:v1 gen:supabase-csharp:v0 gen:supabase-js:v1 gen:supabase-js:v2 gen:realtime",
"gen:api": "npm-run-all gen:api:usage",
@@ -42,6 +43,9 @@
"@mdx-js/react": "^1.6.22",
"@next/mdx": "^12.0.4",
"@radix-ui/react-accordion": "^1.0.1",
"@supabase/auth-helpers-nextjs": "^0.5.4",
"@supabase/auth-helpers-react": "^0.3.1",
"@supabase/supabase-js": "^2.4.1",
"algoliasearch": "^4.14.2",
"babel": "^6.23.0",
"clsx": "^1.2.1",
@@ -52,7 +56,12 @@
"jsrsasign": "^10.5.26",
"lodash": "^4.17.21",
"markdown-toc": "^1.2.0",
"mdast": "^3.0.0",
"mdast-util-from-markdown": "^1.2.0",
"mdast-util-mdx": "^2.0.0",
"mdast-util-to-markdown": "^1.5.0",
"mdx-mermaid": "2.0.0-rc3",
"micromark-extension-mdxjs": "^1.0.0",
"next": "12.3.2",
"next-compose-plugins": "^2.2.1",
"next-mdx-remote": "^4.1.0",
@@ -60,6 +69,7 @@
"next-plugin-yaml": "^1.0.1",
"next-seo": "^5.14.1",
"next-transpile-modules": "^9.0.0",
"openai": "^3.1.0",
"react": "17.0.2",
"react-copy-to-clipboard": "^5.0.2",
"react-dom": "17.0.2",
@@ -72,7 +82,10 @@
"remark-gfm": "^3.0.1",
"remark-slug": "^7.0.1",
"shiki": "^0.11.1",
"sse.js": "^0.6.1",
"ui": "*",
"unist-builder": "^3.0.1",
"unist-util-filter": "^4.0.1",
"valtio": "^1.7.6"
},
"devDependencies": {
@@ -91,6 +104,7 @@
"sass": "^1.55.0",
"ts-node": "^10.9.1",
"tsconfig": "*",
"tsx": "^3.12.2",
"typescript": "^4.5.3"
}
}
+44 -33
View File
@@ -1,21 +1,26 @@
import { createBrowserSupabaseClient } from '@supabase/auth-helpers-nextjs'
import { SessionContextProvider } from '@supabase/auth-helpers-react'
import { ThemeProvider } from 'common/Providers'
import { DefaultSeo } from 'next-seo'
import { useRouter } from 'next/router'
import { useEffect } from 'react'
import { useEffect, useState } from 'react'
import { AppPropsWithLayout } from 'types'
import ClippyProvider from '~/components/Clippy/ClippyProvider'
import { SearchProvider } from '~/components/DocSearch'
import Favicons from '~/components/Favicons'
import { ThemeProvider } from 'common/Providers'
import SiteLayout from '~/layouts/SiteLayout'
import { post } from '~/lib/fetchWrappers'
import FlagProvider from 'components/Flag/FlagProvider'
import '../styles/algolia-search.scss'
import '../styles/ch.scss'
import '../styles/docsearch.scss'
import '../styles/main.scss?v=1.0.0'
import '../styles/new-docs.scss'
import '../styles/prism-okaidia.scss'
import { post } from '~/lib/fetchWrappers'
function MyApp({ Component, pageProps }: AppPropsWithLayout) {
const router = useRouter()
const [supabase] = useState(() => createBrowserSupabaseClient())
function telemetry(route: string) {
return post(`https://api.supabase.io/platform/telemetry/page`, {
@@ -58,36 +63,42 @@ function MyApp({ Component, pageProps }: AppPropsWithLayout) {
return (
<>
<Favicons />
<DefaultSeo
title={SITE_TITLE}
description={SITE_DESCRIPTION}
openGraph={{
type: 'website',
url: 'https://supabase.com/docs',
site_name: SITE_TITLE,
images: [
{
url: `https://supabase.com${basePath}/img/supabase-og-image.png`,
width: 800,
height: 600,
alt: 'Supabase Og Image',
},
],
}}
twitter={{
handle: '@supabase',
site: '@supabase',
cardType: 'summary_large_image',
}}
/>
<ThemeProvider>
<SearchProvider>
<SiteLayout>
<Component {...pageProps} />
</SiteLayout>
</SearchProvider>
</ThemeProvider>
<FlagProvider>
<Favicons />
<DefaultSeo
title={SITE_TITLE}
description={SITE_DESCRIPTION}
openGraph={{
type: 'website',
url: 'https://supabase.com/docs',
site_name: SITE_TITLE,
images: [
{
url: `https://supabase.com${basePath}/img/supabase-og-image.png`,
width: 800,
height: 600,
alt: 'Supabase Og Image',
},
],
}}
twitter={{
handle: '@supabase',
site: '@supabase',
cardType: 'summary_large_image',
}}
/>
<SessionContextProvider supabaseClient={supabase}>
<ThemeProvider>
<SearchProvider>
<ClippyProvider>
<SiteLayout>
<Component {...pageProps} />
</SiteLayout>
</ClippyProvider>
</SearchProvider>
</ThemeProvider>
</SessionContextProvider>
</FlagProvider>
</>
)
}
@@ -152,10 +152,11 @@ Database changes are managed through "migrations." Database migrations are a com
For this guide, create a table called `employees`. In Supabase Studio, navigate to the **SQL Editor** page and run the following SQL command:
```sql
create table employees (
id integer primary key generated always as identity,
name text
);
create table
employees (
id integer primary key generated always as identity,
name text
);
```
<Admonition type="note">
@@ -177,8 +178,9 @@ This creates a new migration named `supabase/migrations/<timestamp>_create_emplo
Use the seed script in `supabase/seed.sql` (created with [`supabase init`](/docs/reference/cli/usage#supabase-init)) to add sample data to the table.
```sql
-- in supabase/seed.sql
insert into public.employees (name)
-- in supabase/seed.sql
insert into
public.employees (name)
values
('Erlich Bachman'),
('Richard Hendricks'),
@@ -198,9 +200,9 @@ You should now see the contents of `employees` in Studio.
Use the [`reset`](/docs/reference/cli/usage#supabase-db-reset) command to revert any changes to the local database.
```sql
-- run on local database to make a change
alter table employees
add department text default 'Hooli';
-- run on local database to make a change
alter table
employees add department text default 'Hooli';
```
Run the following command to reset the local database:
@@ -258,10 +260,6 @@ The local development environment is not as feature-complete as the Supabase Pla
- Logs are not supported through the interface (however you can access them through the Docker containers).
- You cannot update your project settings in the Dashboard—this must be done using the CLI.
```
```
export const Page = ({ children }) => <Layout meta={meta} children={children} />
export default Page
@@ -0,0 +1,101 @@
import Layout from '~/layouts/DefaultGuideLayout'
export const meta = {
id: 'pgvector',
title: 'pgvector: Embeddings and vector similarity',
description:
'pgvector: a PostgreSQL extension for storing embeddings and performing vector similarity search.',
}
[pgvector](https://github.com/pgvector/pgvector/) is a PostgreSQL extension for vector similarity search. It can also be used for storing [embeddings](https://en.wikipedia.org/wiki/Word_embedding).
## Concepts
### Vector similarity
Vector similarity refers to a measure of the similarity between two related items. For example, if you have a list of products, you can use vector similarity to find similar products. To do this, you need to convert each product into a "vector" of numbers, using a mathematical model. You can use a similar model for text, images, and other types of data. Once all of these vectors are stored in the database, you can use vector similarity to find similar items.
### Embeddings
This is particularly useful if you're building on top of OpenAI's [GPT-3](https://openai.com/blog/gpt-3-apps/). You can create and store [embeddings](https://platform.openai.com/docs/guides/embeddings) which match the GPT model you're using.
## Usage
### Enable the extension
<Tabs
scrollable
size="small"
type="underlined"
defaultActiveId="dashboard"
>
<TabPanel id="dashboard" label="Dashboard">
1. Go to the [Database](https://app.supabase.com/project/_/database/tables) page in the Dashboard.
2. Click on **Extensions** in the sidebar.
3. Search for "vector" and enable the extension.
</TabPanel>
<TabPanel id="sql" label="SQL">
```sql
-- Example: enable the "vector" extension.
create extension vector
with
schema extensions;
-- Example: disable the "vector" extension
drop
extension if exists vector;
```
Even though the SQL code is `create extension`, this is the equivalent of "enabling the extension".
To disable an extension, call `drop extension`.
</TabPanel>
</Tabs>
## Usage
### Create a table to store vectors
```sql
create table
posts (
id serial primary key,
title text not null,
body text not null,
embedding vector (1536)
);
```
### Storing a vector / embedding
In this example we'll generate a vector using the OpenAI API client, then store it in the database using the Supabase client.
```js
const title = 'First post!'
const body = 'Hello world!'
// Generate a vector using OpenAI
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input: body,
})
const [responseData] = embeddingResponse.data.data.
// Store the vector in Postgres
const { data, error } = await supabase.from('posts').insert({
title,
body,
embedding: responseData.embedding,
})
```
## Resources
- Source code: [github.com/pgvector/pgvector](https://github.com/pgvector/pgvector/)
export const Page = ({ children }) => <Layout meta={meta} children={children} />
export default Page
@@ -62,7 +62,6 @@ import {
User,
} from '@supabase/supabase-js'
import { environment } from 'src/environments/environment'
import { Database } from 'src/schema'
export interface Profile {
id?: string
@@ -136,7 +135,7 @@ Let's set up an Angular component to manage logins and sign ups. We'll use Magic
Create an **AuthComponent** with `ng g c auth` Angular CLI command.
```ts title=src/app/auth/auth.component.ts
import { Component, OnInit } from '@angular/core'
import { Component } from '@angular/core'
import { FormBuilder } from '@angular/forms'
import { SupabaseService } from '../supabase.service'
@@ -145,7 +144,7 @@ import { SupabaseService } from '../supabase.service'
templateUrl: './auth.component.html',
styleUrls: ['./auth.component.css'],
})
export class AuthComponent implements OnInit {
export class AuthComponent {
loading = false
signInForm = this.formBuilder.group({
@@ -157,8 +156,6 @@ export class AuthComponent implements OnInit {
private readonly formBuilder: FormBuilder
) {}
ngOnInit(): void {}
async onSubmit(): Promise<void> {
try {
this.loading = true
@@ -406,7 +403,7 @@ Let's create an avatar for the user so that they can upload a profile photo.
Create an **AvatarComponent** with `ng g c avatar` Angular CLI command.
```ts title=src/app/avatar/avatar.component.ts
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
import { Component, EventEmitter, Input, Output } from '@angular/core'
import { SafeResourceUrl, DomSanitizer } from '@angular/platform-browser'
import { SupabaseService } from '../supabase.service'
@@ -415,7 +412,7 @@ import { SupabaseService } from '../supabase.service'
templateUrl: './avatar.component.html',
styleUrls: ['./avatar.component.css'],
})
export class AvatarComponent implements OnInit {
export class AvatarComponent {
_avatarUrl: SafeResourceUrl | undefined
uploading = false
@@ -430,8 +427,6 @@ export class AvatarComponent implements OnInit {
constructor(private readonly supabase: SupabaseService, private readonly dom: DomSanitizer) {}
ngOnInit(): void {}
async downloadImage(path: string) {
try {
const { data } = await this.supabase.downLoadImage(path)
+1 -1
View File
@@ -130,7 +130,7 @@ channel.subscribe(async (status) => {
})
if (presenceTrackStatus === 'ok') {
const presenceUntrackStatus = await channe.untrack()
const presenceUntrackStatus = await channel.untrack()
console.log(presenceUntrackStatus)
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 313 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 980 KiB

+335
View File
@@ -0,0 +1,335 @@
import { createClient } from '@supabase/supabase-js'
import { createHash } from 'crypto'
import dotenv from 'dotenv'
import { ObjectExpression } from 'estree'
import { readdir, readFile, stat } from 'fs/promises'
import { Content, Root } from 'mdast'
import { fromMarkdown } from 'mdast-util-from-markdown'
import { mdxFromMarkdown, MdxjsEsm } from 'mdast-util-mdx'
import { toMarkdown } from 'mdast-util-to-markdown'
import { mdxjs } from 'micromark-extension-mdxjs'
import 'openai'
import { Configuration, OpenAIApi } from 'openai'
import { join } from 'path'
import { u } from 'unist-builder'
import { filter } from 'unist-util-filter'
import { inspect } from 'util'
dotenv.config()
const ignoredFiles = ['pages/404.mdx']
/**
* Extracts ES literals from an `estree` `ObjectExpression`
* into a plain JavaScript object.
*/
function getObjectFromExpression(node: ObjectExpression) {
return node.properties.reduce<
Record<string, string | number | bigint | true | RegExp | undefined>
>((object, property) => {
if (property.type !== 'Property') {
return object
}
const key = (property.key.type === 'Identifier' && property.key.name) || undefined
const value = (property.value.type === 'Literal' && property.value.value) || undefined
if (!key) {
return object
}
return {
...object,
[key]: value,
}
}, {})
}
/**
* Extracts the `meta` ESM export from the MDX file.
*
* This info is akin to frontmatter.
*/
function extractMetaExport(mdxTree: Root) {
const metaExportNode = mdxTree.children.find((node): node is MdxjsEsm => {
return (
node.type === 'mdxjsEsm' &&
node.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
node.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
node.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
node.data.estree.body[0].declaration.declarations[0].id.name === 'meta'
)
})
if (!metaExportNode) {
return undefined
}
const objectExpression =
(metaExportNode.data?.estree?.body[0]?.type === 'ExportNamedDeclaration' &&
metaExportNode.data.estree.body[0].declaration?.type === 'VariableDeclaration' &&
metaExportNode.data.estree.body[0].declaration.declarations[0]?.id.type === 'Identifier' &&
metaExportNode.data.estree.body[0].declaration.declarations[0].id.name === 'meta' &&
metaExportNode.data.estree.body[0].declaration.declarations[0].init?.type ===
'ObjectExpression' &&
metaExportNode.data.estree.body[0].declaration.declarations[0].init) ||
undefined
if (!objectExpression) {
return undefined
}
return getObjectFromExpression(objectExpression)
}
/**
* Splits a `mdast` tree into multiple trees based on
* a predicate function. Will include the splitting node
* at the beginning of each tree.
*
* Useful to split a markdown file into smaller sections.
*/
function splitTreeBy(tree: Root, predicate: (node: Content) => boolean) {
return tree.children.reduce<Root[]>((trees, node) => {
const [lastTree] = trees.slice(-1)
if (!lastTree || predicate(node)) {
const tree: Root = u('root', [node])
return trees.concat(tree)
}
lastTree.children.push(node)
return trees
}, [])
}
type Meta = ReturnType<typeof extractMetaExport>
type ProcessedMdx = {
checksum: string
meta: Meta
sections: string[]
}
/**
* Processes MDX content for search indexing.
* It extracts metadata, strips it of all JSX,
* and splits it into sub-sections based on criteria.
*/
function processMdxForSearch(content: string): ProcessedMdx {
const checksum = createHash('sha256').update(content).digest('base64')
const mdxTree = fromMarkdown(content, {
extensions: [mdxjs()],
mdastExtensions: [mdxFromMarkdown()],
})
const meta = extractMetaExport(mdxTree)
// Remove all MDX elements from markdown
const mdTree = filter(
mdxTree,
(node) =>
![
'mdxjsEsm',
'mdxJsxFlowElement',
'mdxJsxTextElement',
'mdxFlowExpression',
'mdxTextExpression',
].includes(node.type)
)
if (!mdTree) {
return {
checksum,
meta,
sections: [],
}
}
const sectionTrees = splitTreeBy(mdTree, (node) => node.type === 'heading')
const sections = sectionTrees.map((tree) => toMarkdown(tree))
return {
checksum,
meta,
sections,
}
}
async function walk(dir: string): Promise<string[]> {
const immediateFiles = await readdir(dir)
const recursiveFiles = await Promise.all(
immediateFiles.map(async (file) => {
const filePath = join(dir, file)
const stats = await stat(filePath)
if (stats.isDirectory()) {
return walk(filePath)
} else if (stats.isFile()) {
return [filePath]
} else {
return []
}
})
)
const flattenedFiles = recursiveFiles.reduce(
(all, folderContents) => all.concat(folderContents),
[]
)
return flattenedFiles
}
async function generateEmbeddings() {
if (
!process.env.NEXT_PUBLIC_SUPABASE_URL ||
!process.env.SUPABASE_SERVICE_ROLE_KEY ||
!process.env.OPENAI_KEY
) {
return console.log(
'Environment variables NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, and OPENAI_KEY are required: skipping embeddings generation'
)
}
const supabaseClient = createClient(
process.env.NEXT_PUBLIC_SUPABASE_URL,
process.env.SUPABASE_SERVICE_ROLE_KEY
)
const markdownFiles = (await walk('pages'))
.filter((fileName) => /\.mdx?$/.test(fileName))
.filter((fileName) => !ignoredFiles.includes(fileName))
console.log(`Discovered ${markdownFiles.length} pages`)
console.log('Checking which pages are new or have changed')
for (const markdownFile of markdownFiles) {
const path = markdownFile.replace(/^pages/, '').replace(/\.mdx?$/, '')
try {
const contents = await readFile(markdownFile, 'utf8')
const { checksum, meta, sections } = processMdxForSearch(contents)
// Check for existing page in DB and compare checksums
const { error: fetchPageError, data: existingPage } = await supabaseClient
.from('page')
.select()
.filter('path', 'eq', path)
.limit(1)
.maybeSingle()
if (fetchPageError) {
throw fetchPageError
}
// We use checksum to determine if this page & its sections need to be regenerated
if (existingPage?.checksum === checksum) {
continue
}
console.log({ checksum, path, meta })
if (existingPage) {
console.log(
`Docs have changed for '${path}', removing old page sections and their embeddings`
)
const { error: deletePageSectionError } = await supabaseClient
.from('page_section')
.delete()
.filter('page_id', 'eq', existingPage.id)
if (deletePageSectionError) {
throw deletePageSectionError
}
}
// Create/update page record. Intentionally clear checksum until we
// have successfully generated all page sections.
const { error: upsertPageError, data: page } = await supabaseClient
.from('page')
.upsert({ checksum: null, path, meta }, { onConflict: 'path' })
.select()
.limit(1)
.single()
if (upsertPageError) {
throw upsertPageError
}
console.log(`Adding ${sections.length} page sections (with embeddings) for '${path}'`)
for (const section of sections) {
// OpenAI recommends replacing newlines with spaces for best results (specific to embeddings)
const input = section.replace(/\n/g, ' ')
try {
const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY })
const openai = new OpenAIApi(configuration)
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input,
})
if (embeddingResponse.status !== 200) {
throw new Error(inspect(embeddingResponse.data, false, 2))
}
const [responseData] = embeddingResponse.data.data
const { error: insertPageSectionError, data: pageSection } = await supabaseClient
.from('page_section')
.insert({
page_id: page.id,
content: section,
token_count: embeddingResponse.data.usage.total_tokens,
embedding: responseData.embedding,
})
.select()
.limit(1)
.single()
if (insertPageSectionError) {
throw insertPageSectionError
}
} catch (err) {
// TODO: decide how to better handle failed embeddings
console.error(
`Failed to generate embeddings for '${path}' page section starting with '${input.slice(
0,
40
)}...'`
)
throw err
}
}
// Set page checksum so that we know this page was stored successfully
const { error: updatePageError } = await supabaseClient
.from('page')
.update({ checksum })
.filter('id', 'eq', page.id)
if (updatePageError) {
throw updatePageError
}
} catch (err) {
console.error(
`Page '${path}' or one/multiple of its page sections failed to store properly. Page has been marked with null checksum to indicate that it needs to be re-generated.`
)
console.error(err)
}
}
console.log('Embedding generation complete')
}
async function main() {
await generateEmbeddings()
}
main().catch((err) => console.error(err))
+1
View File
@@ -953,6 +953,7 @@ body {
// search bar
input[type='search'].DocSearch-Input {
background-color: var(--colors-scale1);
box-shadow: none;
}
.dark .DocSearch-Input {
+12
View File
@@ -0,0 +1,12 @@
declare module 'sse.js' {
export type SSEOptions = EventSourceInit & {
headers?: Record<string, string>
payload?: string
method?: string
}
export class SSE extends EventSource {
constructor(url: string | URL, sseOptions?: SSEOptions)
stream(): void
}
}
@@ -0,0 +1,377 @@
---
title: Storing OpenAI embeddings in Postgres with pgvector
description: An example of how to build an AI-powered search engine using OpenAI's embeddings and PostgreSQL.
author: gregnr
image: embeddings/og_pgvector.png
thumb: embeddings/og_pgvector.png
tags:
- postgres
- ai
date: '2023-01-06'
toc_depth: 3
---
A new PostgreSQL extension is now available in Supabase: [`pgvector`](https://github.com/pgvector/pgvector), an open-source vector similarity search.
The exponential progress of AI functionality over the past year has inspired many new real world applications. One specific challenge has been the ability to store and query _embeddings_ at scale.
In this post we'll explain what embeddings are, why we might want to use them, and how we can store and query them in PostgreSQL using `pgvector`.
## What are embeddings?
Embeddings capture the “relatedness” of text, images, video, or other types of information. This relatedness is most commonly used for:
- **Search:** how similar is a search term to a body of text?
- **Recommendations:** how similar are two products?
- **Classifications:** how do we categorize a body of text?
- **Clustering:** how do we identify trends?
Let's explore an example of text embeddings. Say we have three phrases:
1. “The cat chases the mouse”
2. “The kitten hunts rodents”
3. “I like ham sandwiches”
Your job is to group phrases with similar meaning. If you are a human, this should be obvious. Phrases 1 and 2 are almost identical, while phrase 3 has a completely different meaning.
Although phrases 1 and 2 are similar, they share no common vocabulary (besides “the”). Yet their meanings are nearly identical. How can we teach a computer that these are the same?
## Human language
Humans use words and symbols to communicate language. But words in isolation are mostly meaningless - we need to draw from shared knowledge & experience in order to make sense of them. The phrase “You should Google it” only makes sense if you know that Google is a search engine and that people have been using it as a verb.
In the same way, we need to train a neural network model to understand human language. An effective model should be trained on millions of different examples to understand what each word, phrase, sentence, or paragraph could mean in different contexts.
So how does this relate to embeddings?
## How do embeddings work?
Embeddings compress discrete information (words & symbols) into distributed continuous-valued data (vectors). If we took our phrases from before and plot them on a chart, it might look something like this:
![Vector similarity](/images/blog/embeddings/vector-similarity.png)
Phrases 1 and 2 would be plotted close to each other, since their meanings are similar. We would expect phrase 3 to live somewhere far away since it isn't related. If we had a fourth phrase, “Sally ate Swiss cheese”, this might exist somewhere between phrase 3 (cheese can go on sandwiches) and phrase 1 (mice like Swiss cheese).
In this example we only have 2 dimensions: the X and Y axis. In reality, we would need many more dimensions to effectively capture the complexities of human language.
## OpenAI embeddings
OpenAI offers an [API](https://platform.openai.com/docs/guides/embeddings) to generate embeddings for a string of text using its language model. You feed it any text information (blog articles, documentation, your company's knowledge base), and it will output a vector of floating point numbers that represents the “meaning” of that text.
Compared to our 2-dimensional example above, their latest embedding model `text-embedding-ada-002` will output 1536 dimensions.
Why is this useful? Once we have generated embeddings on multiple texts, it is trivial to calculate how similar they are using vector math operations like cosine distance. A perfect use case for this is search. Your process might look something like this:
1. Pre-process your knowledge base and generate embeddings for each page
2. Store your embeddings to be referenced later (more on this)
3. Build a search page that prompts your user for input
4. Take user's input, generate a one-time embedding, then perform a similarity search against your pre-processed embeddings.
5. Return the most similar pages to the user
## Embeddings in practice
At a small scale, you could store your embeddings in a CSV file, load them into Python, and use a library like `numPy` to calculated similarity between them using something like cosine distance or dot product. OpenAI has a cookbook [example](https://github.com/openai/openai-cookbook/blob/main/examples/Semantic_text_search_using_embeddings.ipynb) that does just that. Unfortunately this likely won't scale well:
- What if I need to store and search over a large number of documents and embeddings (more than can fit in memory)?
- What if I want to create/update/delete embeddings dynamically?
- What if I'm not using Python?
### Using PostgreSQL
Enter [`pgvector`](https://github.com/pgvector/pgvector), an extension for PostgreSQL that allows you to both store and query vector embeddings within your database. Let's try it out.
First we'll enable the **Vector** extension. In Supabase, this can be done from the web portal through `Database` → `Extensions`. You can also do this in SQL by running:
```sql
create extension vector;
```
Next let's create a table to store our documents and their embeddings:
```sql
create table documents (
id bigserial primary key,
content text,
embedding vector (1536)
);
```
`pgvector` introduces a new data type called `vector`. In the code above, we create a column named `embedding` with the `vector` data type. The size of the vector defines how many dimensions the vector holds. OpenAI's `text-embedding-ada-002` model outputs 1536 dimensions, so we will use that for our vector size.
We also create a `text` column named `content` to store the original document text that produced this embedding. Depending on your use case, you might just store a reference (URL or foreign key) to a document here instead.
Soon we're going to need to perform a similarity search over these embeddings. Let's create a function to do that:
```sql
create or replace function match_documents (
query_embedding vector(1536),
similarity_threshold float,
match_count int
)
returns table (
id bigint,
content text,
similarity float
)
language plpgsql
as $$
begin
return query
select
id,
content,
(documents.embedding <=> query_embedding) as similarity
from documents
where (documents.embedding <=> query_embedding) > similarity_threshold
order by documents.embedding <=> query_embedding
limit match_count;
end;
$$;
```
`pgvector` introduces 3 new operators that can be used to calculate similarity:
| Operator | Description |
| -------- | ---------------------- |
| `<->` | Euclidean distance |
| `<#>` | negative inner product |
| `<=>` | cosine distance |
OpenAI recommends cosine similarity on their embeddings, so we will use that here.
Now we can call `match_documents()`, pass in our embedding, similarity threshold, and match count, and we'll get a list of all documents that match. And since this is all managed by Postgres, our application code becomes very simple.
### Indexing
Once your table starts to grow with embeddings, you will likely want to add an index to speed up queries. Vector indexes are particularly important when you're ordering results because vectors are not grouped by similarity, so finding the closest by sequential scan is a resource-intensive operation.
Each distance operator requires a different type of index. We expect to order by cosine distance, so we need `vector_cosine_ops` index. A good starting number of lists is 4 \* sqrt(table_rows):
```sql
create index on documents
using ivfflat (embedding vector_cosine_ops)
with (lists = 100);
```
You can read more about indexing on `pgvector`'s GitHub page [here](https://github.com/pgvector/pgvector#indexing).
### Generating embeddings
Let's use JavaScript to generate embeddings and store them in Postgres:
```tsx
import { createClient } from '@supabase/supabase-js'
import { Configuration, OpenAIApi } from 'openai'
import { supabaseClient } from './lib/supabase'
async function generateEmbeddings() {
const configuration = new Configuration({ apiKey: '<YOUR_OPENAI_KEY>' })
const openAi = new OpenAIApi(configuration)
const documents = await getDocuments() // Your custom function to load docs
// Assuming each document is a string
for (const document of documents) {
// OpenAI recommends replacing newlines with spaces for best results
const input = document.replace(/\n/g, ' ')
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input,
})
const [{ embedding }] = embeddingResponse.data.data
// In production we should handle possible errors
await supabaseClient.from('documents').insert({
content: document,
embedding,
})
}
}
```
### Building a simple search function
Finally, let's create an [Edge Function](https://supabase.com/docs/guides/functions) to perform our similarity search:
```tsx
import { serve } from 'https://deno.land/std@0.170.0/http/server.ts'
import 'https://deno.land/x/xhr@0.2.1/mod.ts'
import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.5.0'
import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
import { supabaseClient } from './lib/supabase'
export const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
}
serve(async (req) => {
// Handle CORS
if (req.method === 'OPTIONS') {
return new Response('ok', { headers: corsHeaders })
}
// Search query is passed in request payload
const { query } = await req.json()
// OpenAI recommends replacing newlines with spaces for best results
const input = query.replace(/\n/g, ' ')
const configuration = new Configuration({ apiKey: '<YOUR_OPENAI_KEY>' })
const openai = new OpenAIApi(configuration)
// Generate a one-time embedding for the query itself
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input,
})
const [{ embedding }] = embeddingResponse.data.data
// In production we should handle possible errors
const { data: documents } = await supabaseClient.rpc('match_documents', {
query_embedding: embedding,
match_threshold: 0.78, // Choose an appropriate threshold for your data
match_count: 10, // Choose the number of matches
})
return new Response(JSON.stringify(documents), {
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
})
})
```
### Building a smarter search function
ChatGPT doesn't just return existing documents. It's able to assimilate a variety of information into a single, cohesive answer. To do this, we need to provide GPT with some relevant documents, and a prompt that it can use to formulate this answer.
One of the biggest challenges of OpenAI's `text-davinci-003` [completion model](https://beta.openai.com/docs/guides/completion) is the 4000 token limit. You must fit both your prompt and the resulting completion within the 4000 tokens. This makes it challenging if you wanted to prompt GPT-3 to answer questions about your own custom knowledge base that would never fit in a single prompt.
Embeddings can help solve this by splitting your prompts into a two-phased process:
1. Query your embedding database for the most relevant documents related to the question
2. Inject these documents as context for GPT-3 to reference in its answer
Here's another Edge Function that expands upon the simple example above:
```tsx
import { serve } from 'https://deno.land/std@0.170.0/http/server.ts'
import 'https://deno.land/x/xhr@0.2.1/mod.ts'
import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.5.0'
import GPT3Tokenizer from 'https://esm.sh/gpt3-tokenizer@1.1.5'
import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
import { stripIndent } from 'https://esm.sh/common-tags@1.8.2'
import { supabaseClient } from './lib/supabase'
export const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
}
serve(async (req) => {
// Handle CORS
if (req.method === 'OPTIONS') {
return new Response('ok', { headers: corsHeaders })
}
// Search query is passed in request payload
const { query } = await req.json()
// OpenAI recommends replacing newlines with spaces for best results
const input = query.replace(/\n/g, ' ')
const configuration = new Configuration({ apiKey: '<YOUR_OPENAI_KEY>' })
const openai = new OpenAIApi(configuration)
// Generate a one-time embedding for the query itself
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input,
})
const [{ embedding }] = embeddingResponse.data.data
// Fetching whole documents for this simple example.
//
// Ideally for context injection, documents are chunked into
// smaller sections at earlier pre-processing/embedding step.
const { data: documents } = await supabaseClient.rpc('match_documents', {
query_embedding: embedding,
match_threshold: 0.78, // Choose an appropriate threshold for your data
match_count: 10, // Choose the number of matches
})
const tokenizer = new GPT3Tokenizer({ type: 'gpt3' })
let tokenCount = 0
let contextText = ''
// Concat matched documents
for (let i = 0; i < documents.length; i++) {
const document = documents[i]
const content = document.content
const encoded = tokenizer.encode(content)
tokenCount += encoded.text.length
// Limit context to max 1500 tokens (configurable)
if (tokenCount > 1500) {
break
}
contextText += `${content.trim()}\n---\n`
}
const prompt = stripIndent`${oneLine`
You are a very enthusiastic Supabase representative who loves
to help people! Given the following sections from the Supabase
documentation, answer the question using only that information,
outputted in markdown format. If you are unsure and the answer
is not explicitly written in the documentation, say
"Sorry, I don't know how to help with that."`}
Context sections:
${contextText}
Question: """
${query}
"""
Answer as markdown (including related code snippets if available):
`
// In production we should handle possible errors
const completionResponse = await openai.createCompletion({
model: 'text-davinci-003',
prompt,
max_tokens: 512, // Choose the max allowed tokens in completion
temperature: 0, // Set to 0 for deterministic results
})
const {
id,
choices: [{ text }],
} = completionResponse.data
return new Response(JSON.stringify({ id, text }), {
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
})
})
```
### Streaming results
OpenAI API responses take longer to depending on the length of the “answer”. ChatGPT has a nice UX for this by streaming the response to the user immediately. You can see a similar effect for the Supabase docs:
<video width="99%" autoPlay="" loop="" muted="" playsInline="" controls="true">
<source src="/images/blog/embeddings/embeddings.mp4" type="video/mp4" />
</video>
The OpenAI API supports [completion streaming](https://platform.openai.com/docs/api-reference/completions/create#completions/create-stream) with Server Side Events. Supabase Edge Functions are run Deno, which also supports [Server Side Events](https://deno.com/blog/deploy-streams#server-sent-events). Check out [this commit](https://github.com/supabase/supabase/pull/12056/commits/bd83e9ba2f7263440888228e3b29007604d94841) to see how we modified the Function above to build a streaming interface.
## Wrap up
Storing embeddings in Postgres opens a world of possibilities. You can combine your search function with telemetry functions, add an user-provided feedback (thumbs up/down), and make your search feel more integrated with your products.
Tomorrow we'll be releasing our own version of this interface in the Supabase docs, to provide Supabase-specific answers.
The pgvector extension is available on all new Supabase projects today. If you want to try it out, launch a new Postgres database today: [database.new](https://database.new)
+7
View File
@@ -264,5 +264,12 @@
"position": "Engineering",
"author_url": "https://github.com/hf",
"author_image_url": "https://github.com/hf.png"
},
{
"author_id": "gregnr",
"author": "Greg Richardson",
"position": "Open Source Contributor",
"author_url": "https://github.com/gregnr",
"author_image_url": "https://github.com/gregnr.png"
}
]
Binary file not shown.

After

Width:  |  Height:  |  Size: 403 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 478 KiB

+161 -154
View File
File diff suppressed because it is too large. Load diff
+1069 -135
View File
File diff suppressed because it is too large. Load diff
+19 -8
View File
@@ -10,14 +10,17 @@
"id": "installing",
"slug": "installing",
"type": "markdown",
"excludes": ["reference_javascript_v1", "reference_dart_v0", "reference_python_v2"]
"excludes": [
"reference_javascript_v1",
"reference_dart_v0",
"reference_python_v2"
]
},
{
"title": "Initializing",
"id": "initializing",
"slug": "initializing",
"type": "markdown",
"libs": ["csharp"]
"type": "function"
},
{
"title": "Typescript Support",
@@ -355,6 +358,14 @@
"product": "database",
"parent": "modifiers",
"type": "function"
},
{
"id": "returns",
"title": "Override type of successful response",
"slug": "db-returns",
"product": "database",
"parent": "modifiers",
"type": "function"
}
]
}
@@ -367,7 +378,6 @@
"id": "auth-api",
"title": "Overview",
"slug": "auth-api",
"type": "function"
},
{
@@ -570,7 +580,6 @@
"id": "admin-api",
"title": "Auth Admin",
"slug": "admin-api",
"type": "function",
"items": [
{
@@ -704,7 +713,6 @@
"product": "realtime",
"type": "function"
},
{
"id": "remove-channel",
"title": "Unsubscribe from a channel",
@@ -847,7 +855,10 @@
},
{
"title": "Misc",
"excludes": ["reference_dart_v1", "reference_dart_v0"],
"excludes": [
"reference_dart_v1",
"reference_dart_v0"
],
"items": [
{
"title": "Release Notes",
@@ -858,4 +869,4 @@
}
]
}
]
]
+98
View File
@@ -16,6 +16,104 @@ info:
version: '0.8.0'
functions:
- id: initializing
title: Initializing
description: |
Initializing a new client is pretty straightforward. Find your project url and public key from the
admin panel and pass it into your client initialization function.
`supabase-csharp` is heavily dependent on Models deriving from `BaseModel`. To interact with the API, one must have the associated model (see example) specified.
Leverage `Table`, `PrimaryKey`, and `Column` attributes to specify names of classes/properties that are different from their C# Versions.
examples:
- id: csharp-init-standard
name: Standard
code: |
```c#
var url = Environment.GetEnvironmentVariable("SUPABASE_URL");
var key = Environment.GetEnvironmentVariable("SUPABASE_KEY");
var options = new Supabase.SupabaseOptions
{
AutoConnectRealtime = true
};
var supabase = new Supabase.Client(url, key, options);
await supabase.InitializeAsync();
```
- id: csharp-init-maui
name: Dependency Injection (Maui-like)
code: |
```c#
public static MauiApp CreateMauiApp()
{
// ...
var builder = MauiApp.CreateBuilder();
var url = Environment.GetEnvironmentVariable("SUPABASE_URL");
var key = Environment.GetEnvironmentVariable("SUPABASE_KEY");
var options = new SupabaseOptions
{
AutoRefreshToken = true,
AutoConnectRealtime = true,
SessionHandler = new SupabaseSessionHandler()
};
// Note the creation as a singleton.
builder.Services.AddSingleton(provider => new Supabase.Client(url, key, options));
}
```
- id: csharp-init-showing-models
name: With Models Example
code: |
```c#
// Given the following Model representing the Supabase Database (Message.cs)
[Table("messages")]
public class Message : BaseModel
{
[PrimaryKey("id")]
public int Id { get; set; }
[Column("username")]
public string UserName { get; set; }
[Column("channel_id")]
public int ChannelId { get; set; }
public override bool Equals(object obj)
{
return obj is Message message &&
Id == message.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
}
void Initialize()
{
// Get All Messages
var response = await client.Table<Message>().Get();
List<Message> models = response.Models;
// Insert
var newMessage = new Message { UserName = "acupofjose", ChannelId = 1 };
await client.Table<Message>().Insert();
// Update
var model = response.Models.First();
model.UserName = "elrhomariyounes";
await model.Update();
// Delete
await response.Models.Last().Delete();
// etc.
}
```
- id: sign-up
title: 'SignUp()'
description: |
+25 -5
View File
@@ -66,9 +66,9 @@ functions:
db: { schema: 'other_schema' }
})
```
description: |
By default the API server points to the `public` schema. You can enable other database schemas within the Dashboard.
Go to `Settings > API > Schema` and add the schema which you want to expose to the API.
Go to [Settings > API > Exposed schemas](https://app.supabase.com/project/_/settings/api) and add the schema which you want to expose to the API.
Note: each client connection can only access a single schema, so the code above can access the `other_schema` schema but cannot access the `public` schema.
- id: custom-fetch-implementation
@@ -81,7 +81,7 @@ functions:
global: { fetch: fetch.bind(globalThis) }
})
```
description: |
`supabase-js` uses the [`cross-fetch`](https://www.npmjs.com/package/cross-fetch) library to make HTTP requests,
but an alternative `fetch` implementation can be provided as an option.
This is most useful in environments where `cross-fetch` is not compatible (for instance Cloudflare Workers).
@@ -101,7 +101,7 @@ functions:
},
});
```
description: |
For React Native we recommend using `AsyncStorage` as the storage implementation for Supabase Auth.
- id: auth-api
title: 'Overview'
@@ -3895,6 +3895,25 @@ functions:
By default, the data is returned in JSON format, but can also be returned as Comma Separated Values.
hideCodeBlock: true
isSpotlight: true
- id: returns
$ref: '@supabase/postgrest-js.PostgrestTransformBuilder.returns'
title: returns()
examples:
- id: override-type-of-successful-response
name: Override type of successful response
code: |
```ts
const { data } = await supabase
.from('countries')
.select()
.returns<MyType>()
```
response: |
```
let x: typeof data // MyType | null
```
hideCodeBlock: true
isSpotlight: true
# NOTE: Not available on currently deployed PostgREST
# db.geojson():
@@ -4421,7 +4440,8 @@ functions:
.from('avatars')
.update('public/avatar1.png', avatarFile, {
cacheControl: '3600',
upsert: false
// Overwrite file if it exists
upsert: true
})
```
- id: update-file-using-arraybuffer-from-base64-file-data
+6 -7
View File
@@ -498,21 +498,20 @@ export default class MetaStore implements IMetaStore {
}
) {
const { duplicateTable, isRLSEnabled, isRealtimeEnabled, isDuplicateRows } = metadata
const sourceTableName = duplicateTable.name
const { name: sourceTableName, schema: sourceTableSchema } = duplicateTable
const duplicatedTableName = payload.name
// The following query will copy the structure of the table along with indexes, constraints and
// triggers. However, foreign key constraints are not duplicated over - has to be done separately
const table = await this.rootStore.meta.query(
`CREATE TABLE "${duplicatedTableName}" (LIKE "${sourceTableName}" INCLUDING ALL);`
`CREATE TABLE "${sourceTableSchema}"."${duplicatedTableName}" (LIKE "${sourceTableSchema}"."${sourceTableName}" INCLUDING ALL);`
)
if (table.error) throw table.error
// Duplicate foreign key constraints over
const relationships = duplicateTable.relationships
if (relationships.length > 0) {
// @ts-ignore, but might need to investigate, sounds bad:
// Type instantiation is excessively deep and possibly infinite
// @ts-ignore
relationships.map(async (relationship: PostgresRelationship) => {
const relation = await this.rootStore.meta.addForeignKey({
...relationship,
@@ -525,7 +524,7 @@ export default class MetaStore implements IMetaStore {
// Duplicate rows if needed
if (isDuplicateRows) {
const rows = await this.rootStore.meta.query(
`INSERT INTO "${duplicatedTableName}" SELECT * FROM ${sourceTableName};`
`INSERT INTO "${sourceTableSchema}"."${duplicatedTableName}" SELECT * FROM "${sourceTableSchema}"."${sourceTableName}";`
)
if (rows.error) throw rows.error
@@ -535,7 +534,7 @@ export default class MetaStore implements IMetaStore {
const identityColumns = columns.filter((column) => column.identity_generation !== null)
identityColumns.map(async (column) => {
const identity = await this.rootStore.meta.query(
`SELECT setval('${duplicatedTableName}_${column.name}_seq', (SELECT MAX("${column.name}") FROM "${sourceTableName}"));`
`SELECT setval('"${sourceTableSchema}"."${duplicatedTableName}_${column.name}_seq"', (SELECT MAX("${column.name}") FROM "${sourceTableSchema}"."${sourceTableName}"));`
)
if (identity.error) throw identity.error
})
@@ -543,7 +542,7 @@ export default class MetaStore implements IMetaStore {
await this.tables.load()
const tables = this.tables.list()
const duplicatedTable = find(tables, { name: duplicatedTableName })
const duplicatedTable = find(tables, { schema: sourceTableSchema, name: duplicatedTableName })
if (isRLSEnabled) {
const updateTable: any = await this.tables.update(duplicatedTable!.id, {
+72
View File
@@ -0,0 +1,72 @@
# A string used to distinguish different Supabase projects on the same host. Defaults to the working
# directory name when running `supabase init`.
project_id = "supabase"
[api]
# Port to use for the API URL.
port = 54321
# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
# endpoints. public and storage are always included.
schemas = ["public", "storage", "graphql_public"]
# Extra schemas to add to the search_path of every request. public is always included.
extra_search_path = ["public", "extensions"]
# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
# for accidental or malicious requests.
max_rows = 1000
[db]
# Port to use for the local database URL.
port = 54322
# The database major version to use. This has to be the same as your remote database's. Run `SHOW
# server_version;` on the remote database to check.
major_version = 15
[studio]
# Port to use for Supabase Studio.
port = 54323
# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
# are monitored, and you can view the emails that would have been sent from the web interface.
[inbucket]
# Port to use for the email testing server web interface.
port = 54324
smtp_port = 54325
pop3_port = 54326
[storage]
# The maximum file size allowed (e.g. "5MB", "500KB").
file_size_limit = "50MiB"
[auth]
# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
# in emails.
site_url = "http://localhost:3000"
# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
additional_redirect_urls = ["https://localhost:3000"]
# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 seconds (one
# week).
jwt_expiry = 3600
# Allow/disallow new user signups to your project.
enable_signup = true
[auth.email]
# Allow/disallow new user signups via email to your project.
enable_signup = true
# If enabled, a user will be required to confirm any email change on both the old, and new email
# addresses. If disabled, only the new email is required to confirm.
double_confirm_changes = true
# If enabled, users need to confirm their email address before signing in.
enable_confirmations = false
# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin`, `notion`, `twitch`,
# `twitter`, `slack`, `spotify`, `workos`, `zoom`.
[auth.external.apple]
enabled = false
client_id = ""
secret = ""
# Overrides the default auth redirectUrl.
redirect_uri = ""
# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure,
# or any other third-party OIDC providers.
url = ""
@@ -0,0 +1,7 @@
export class ApplicationError extends Error {
constructor(message: string, public data: Record<string, any> = {}) {
super(message)
}
}
export class UserError extends ApplicationError {}
+187
View File
@@ -0,0 +1,187 @@
import { serve } from 'https://deno.land/std@0.170.0/http/server.ts'
import 'https://deno.land/x/xhr@0.2.1/mod.ts'
import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.5.0'
import { oneLine, stripIndent } from 'https://esm.sh/common-tags@1.8.2'
import GPT3Tokenizer from 'https://esm.sh/gpt3-tokenizer@1.1.5'
import { Configuration, CreateCompletionRequest, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
import { ApplicationError, UserError } from './errors.ts'
const openAiKey = Deno.env.get('OPENAI_KEY')
const supabaseUrl = Deno.env.get('SUPABASE_URL')
const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')
export const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
}
serve(async (req) => {
try {
// Handle CORS
if (req.method === 'OPTIONS') {
return new Response('ok', { headers: corsHeaders })
}
if (!openAiKey) {
throw new ApplicationError('Missing environment variable OPENAI_KEY')
}
if (!supabaseUrl) {
throw new ApplicationError('Missing environment variable SUPABASE_URL')
}
if (!supabaseServiceKey) {
throw new ApplicationError('Missing environment variable SUPABASE_SERVICE_ROLE_KEY')
}
const requestData = await req.json()
if (!requestData) {
throw new UserError('Missing request data')
}
const { query } = requestData
if (!query) {
throw new UserError('Missing query in request data')
}
const sanitizedQuery = query.trim()
const supabaseClient = createClient(supabaseUrl, supabaseServiceKey)
const configuration = new Configuration({ apiKey: openAiKey })
const openai = new OpenAIApi(configuration)
// Moderate the content to comply with OpenAI T&C
const moderationResponse = await openai.createModeration({ input: sanitizedQuery })
const [results] = moderationResponse.data.results
if (results.flagged) {
throw new UserError('Flagged content', {
flagged: true,
categories: results.categories,
})
}
const embeddingResponse = await openai.createEmbedding({
model: 'text-embedding-ada-002',
input: sanitizedQuery.replaceAll('\n', ' '),
})
if (embeddingResponse.status !== 200) {
throw new ApplicationError('Failed to create embedding for question', embeddingResponse)
}
const [{ embedding }] = embeddingResponse.data.data
const { error: matchError, data: pageSections } = await supabaseClient.rpc(
'match_page_sections',
{
embedding,
match_threshold: 0.78,
match_count: 10,
min_content_length: 50,
}
)
if (matchError) {
throw new ApplicationError('Failed to match page sections', matchError)
}
const tokenizer = new GPT3Tokenizer({ type: 'gpt3' })
let tokenCount = 0
let contextText = ''
for (let i = 0; i < pageSections.length; i++) {
const pageSection = pageSections[i]
const content = pageSection.content
const encoded = tokenizer.encode(content)
tokenCount += encoded.text.length
if (tokenCount >= 1500) {
break
}
contextText += `${content.trim()}\n---\n`
}
const prompt = stripIndent`
${oneLine`
You are a very enthusiastic Supabase representative who loves
to help people! Given the following sections from the Supabase
documentation, answer the question using only that information,
outputted in markdown format. If you are unsure and the answer
is not explicitly written in the documentation, say
"Sorry, I don't know how to help with that."
`}
Context sections:
${contextText}
Question: """
${sanitizedQuery}
"""
Answer as markdown (including related code snippets if available):
`
const completionOptions: CreateCompletionRequest = {
model: 'text-davinci-003',
prompt,
max_tokens: 512,
temperature: 0,
stream: true,
}
const response = await fetch('https://api.openai.com/v1/completions', {
headers: {
Authorization: `Bearer ${openAiKey}`,
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(completionOptions),
})
if (!response.ok) {
const error = await response.json()
throw new ApplicationError('Failed to generate completion', error)
}
// Proxy the streamed SSE response from OpenAI
return new Response(response.body, {
headers: {
...corsHeaders,
'Content-Type': 'text/event-stream',
},
})
} catch (err: unknown) {
if (err instanceof UserError) {
return new Response(
JSON.stringify({
error: err.message,
data: err.data,
}),
{
status: 400,
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
}
)
}
// Print out unexpected errors to help with debugging
console.error(err)
// TODO: include more response info in debug environments
return new Response(
JSON.stringify({
error: 'There was an error processing your request',
}),
{
status: 500,
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
}
)
}
})
@@ -0,0 +1,16 @@
create extension if not exists vector with schema public;
create table "public"."page" (
id bigserial primary key,
path text not null unique,
checksum text,
meta jsonb
);
create table "public"."page_section" (
id bigserial primary key,
page_id bigint not null references public.page on delete cascade,
content text,
token_count int,
embedding vector(1536)
);
@@ -0,0 +1,31 @@
create or replace function match_page_sections(embedding vector(1536), match_threshold float, match_count int, min_content_length int)
returns table (path text, content text, similarity float)
language plpgsql
as $$
#variable_conflict use_variable
begin
return query
select
page.path,
page_section.content,
(page_section.embedding <#> embedding) * -1 as similarity
from page_section
join page
on page_section.page_id = page.id
-- We only care about sections that have a useful amount of content
where length(page_section.content) >= min_content_length
-- The dot product is negative because of a Postgres limitation, so we negate it
and (page_section.embedding <#> embedding) * -1 > match_threshold
-- OpenAI embeddings are normalized to length 1, so
-- cosine similarity and dot product will produce the same results.
-- Using dot product which can be computed slightly faster.
--
-- For the different syntaxes, see https://github.com/pgvector/pgvector
order by page_section.embedding <#> embedding
limit match_count;
end;
$$;
View File
Whitespace-only changes.