Add option to add web content

apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx

@@ -3,7 +3,7 @@ import { AnimatePresence, motion } from 'framer-motion'
 import { last } from 'lodash'
 import { ExternalLink, FileText, MessageCircleMore, Plus, WandSparkles } from 'lucide-react'
 import Link from 'next/link'
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { FormEvent, useEffect, useMemo, useRef, useState } from 'react'
 import { toast } from 'sonner'
 
 import type { Message as MessageType } from 'ai/react'
@@ -38,6 +38,7 @@ import {
   DropdownMenuSubContent,
   DropdownMenuSubTrigger,
   DropdownMenuTrigger,
+  Input_Shadcn_,
   SheetHeader,
   SheetSection,
   Tooltip_Shadcn_,
@@ -98,6 +99,8 @@ export const AIAssistant = ({
   const [assistantError, setAssistantError] = useState<string>()
   const [lastSentMessage, setLastSentMessage] = useState<MessageType>()
   const [isConfirmOptInModalOpen, setIsConfirmOptInModalOpen] = useState(false)
+  const [webUrl, setWebUrl] = useState('')
+  const [webContent, setWebContent] = useState('')
 
   const docsUrl = retrieveDocsUrl(selectedDatabaseEntity as SupportedAssistantEntities)
   const entityContext = ASSISTANT_SUPPORT_ENTITIES.find((x) => x.id === selectedDatabaseEntity)
@@ -134,7 +137,7 @@ export const AIAssistant = ({
   } = useChat({
     id,
     api: `${BASE_PATH}/api/ai/sql/generate-v2`,
-    body: { entityDefinitions, context: selectedDatabaseEntity },
+    body: { entityDefinitions, context: selectedDatabaseEntity, webContent },
     onError: (error) => setAssistantError(JSON.parse(error.message).error),
   })
 
@@ -275,6 +278,35 @@ export const AIAssistant = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [messages.length])
 
+  async function handleWebSubmit(e: FormEvent<HTMLFormElement>) {
+    e.preventDefault()
+
+    if (!webUrl) return
+
+    try {
+      const response = await fetch(`${BASE_PATH}/api/ai/web-url/get-url-content`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ url: webUrl }),
+      })
+
+      if (!response.ok) {
+        throw new Error('Failed to fetch web content')
+      }
+
+      const data = await response.json()
+      console.log('Fetched web content:', data.content)
+
+      setWebContent(data.content)
+    } catch (error) {
+      console.error('Error fetching web content:', error)
+      toast.error('Failed to fetch web content. Please try again.')
+    }
+
+    // setWebUrl('')
+  }
   return (
     <>
       <div className={cn('flex flex-col', className)}>
@@ -508,6 +540,27 @@ export const AIAssistant = ({
                               </DropdownMenuSubContent>
                             </DropdownMenuSub>
                           )}
+                          <DropdownMenuSub>
+                            <DropdownMenuSubTrigger className="gap-x-2">
+                              <div className="flex flex-col gap-y-1">
+                                <p>Web</p>
+                                <p className="text-foreground-lighter">
+                                  Add a link to a website to help the assistant answer your question
+                                </p>
+                              </div>
+                            </DropdownMenuSubTrigger>
+                            <DropdownMenuSubContent className="p-0 w-72">
+                              <div className="flex flex-col gap-y-1 p-4 ">
+                                <form onSubmit={handleWebSubmit}>
+                                  <Input_Shadcn_
+                                    placeholder="https://example.com/docs"
+                                    value={webUrl}
+                                    onChange={(e) => setWebUrl(e.target.value)}
+                                  />
+                                </form>
+                              </div>
+                            </DropdownMenuSubContent>
+                          </DropdownMenuSub>
                         </>
                       )}
                     </DropdownMenuContent>
@@ -574,6 +627,25 @@ export const AIAssistant = ({
                       }
                     />
                   )}
+                  {webUrl.length > 0 && (
+                    <ContextBadge
+                      label="Web"
+                      value={webContent.length > 0 ? webContent.slice(0, 20) + '...' : webUrl}
+                      onRemove={() => setWebUrl('')}
+                      tooltip={
+                        webContent.length > 20 ? (
+                          <>
+                            <p className="text-foreground-light">
+                              {webContent.length} site selected:
+                            </p>
+                            <ul className="list-disc pl-4">
+                              <li>{webUrl}</li>
+                            </ul>
+                          </>
+                        ) : undefined
+                      }
+                    />
+                  )}
                 </div>
 
                 <AssistantChatForm

apps/studio/pages/api/ai/sql/generate-v2.ts

@@ -8,7 +8,6 @@ export const config = {
   runtime: 'edge',
   /* To avoid OpenAI errors, restrict to the Vercel Edge Function regions that
   overlap with the OpenAI API regions.
-  
   Reference for Vercel regions: https://vercel.com/docs/edge-network/regions#region-list
   Reference for OpenAI regions: https://help.openai.com/en/articles/5347006-openai-api-supported-countries-and-territories
   */
@@ -72,12 +71,20 @@ async function handlePost(request: NextRequest) {
     existingSql?: string
     entityDefinitions: string[]
     context?: SupportedAssistantEntities
+    webContent?: string
   }>)
 
-  const { messages, existingSql, entityDefinitions, context } = body
+  const { messages, existingSql, entityDefinitions, context, webContent } = body
 
   try {
-    const stream = await chatSql(openai, messages, existingSql, entityDefinitions, context)
+    const stream = await chatSql(
+      openai,
+      messages,
+      existingSql,
+      entityDefinitions,
+      context,
+      webContent
+    )
     return new StreamingTextResponse(stream)
   } catch (error) {
     if (error instanceof Error) {

apps/studio/pages/api/ai/web-url/get-url-content.ts

@@ -0,0 +1,121 @@
+import { NextApiRequest, NextApiResponse } from 'next'
+import apiWrapper from 'lib/api/apiWrapper'
+import * as cheerio from 'cheerio'
+
+export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler)
+
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== 'POST') {
+    return res.status(405).json({ error: { message: 'Method Not Allowed' } })
+  }
+
+  const { url } = req.body
+
+  if (!url) {
+    return res.status(400).json({ error: { message: 'URL is required' } })
+  }
+
+  try {
+    const response = await fetch(url)
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`)
+    }
+    const html = await response.text()
+    const $ = cheerio.load(html)
+
+    // Extract title
+    const title = $('title').text().trim()
+
+    // Extract meta description
+    const description = $('meta[name="description"]').attr('content')?.trim() || ''
+
+    // Extract main content
+    const mainContent = extractMainContent($)
+
+    return res.status(200).json({
+      title,
+      description,
+      content: mainContent,
+      url,
+    })
+  } catch (error) {
+    console.error('Error fetching web content:', error)
+    return res.status(500).json({ error: { message: 'Failed to fetch web content' } })
+  }
+}
+
+function extractMainContent($: cheerio.CheerioAPI): string {
+  // Remove script tags, style tags, and comments
+  $('script, style, comment').remove()
+
+  // Common selectors for main content
+  const contentSelectors = [
+    'article',
+    '[role="main"]',
+    '.main-content',
+    '#main-content',
+    '.post-content',
+    '.entry-content',
+    '.content',
+    '#content',
+    'main',
+  ]
+
+  let mainContent = ''
+
+  // Try to find content using common selectors
+  for (const selector of contentSelectors) {
+    const element = $(selector)
+    if (element.length > 0) {
+      mainContent = element.text().trim()
+      break
+    }
+  }
+
+  // If no content found, use a density-based approach
+  if (!mainContent) {
+    mainContent = findContentByDensity($)
+  }
+
+  // Fallback to body content if still empty
+  if (!mainContent) {
+    mainContent = $('body').text().trim()
+  }
+
+  // Limit content length and clean it up
+  return cleanAndLimitContent(mainContent)
+}
+
+function findContentByDensity($: cheerio.CheerioAPI): string {
+  let bestElement: cheerio.Element | null = null
+  let maxScore = 0
+
+  $('p')
+    .parent()
+    .each((_, element) => {
+      const $element = $(element)
+      const text = $element.text()
+      const linkDensity = $element.find('a').text().length / text.length || 0
+      const score = text.length * (1 - linkDensity)
+
+      if (score > maxScore) {
+        maxScore = score
+        bestElement = element
+      }
+    })
+
+  return bestElement ? $(bestElement).text().trim() : ''
+}
+
+function cleanAndLimitContent(content: string): string {
+  // Remove extra whitespace and newlines
+  content = content.replace(/\s+/g, ' ').trim()
+
+  // Limit to around 1000 words
+  const words = content.split(' ')
+  if (words.length > 1000) {
+    content = words.slice(0, 1000).join(' ') + '...'
+  }
+
+  return content
+}

package-lock.json

@@ -14,6 +14,9 @@
         "playwright-tests",
         "packages/*"
       ],
+      "dependencies": {
+        "@types/cheerio": "^0.22.35"
+      },
       "devDependencies": {
         "eslint": "^8.57.0",
         "prettier": "^4.0.0-alpha.8",
@@ -16452,6 +16455,14 @@
       "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.2.tgz",
       "integrity": "sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg=="
     },
+    "node_modules/@types/cheerio": {
+      "version": "0.22.35",
+      "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz",
+      "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/classnames": {
       "version": "2.3.1",
       "dev": true,

package.json

@@ -5,7 +5,12 @@
   "author": "Supabase, Inc.",
   "license": "Apache-2.0",
   "private": true,
-  "workspaces": ["apps/*", "tests", "playwright-tests", "packages/*"],
+  "workspaces": [
+    "apps/*",
+    "tests",
+    "playwright-tests",
+    "packages/*"
+  ],
   "scripts": {
     "build": "turbo run build",
     "build:studio": "turbo run build --filter=studio",
@@ -53,6 +58,16 @@
     "npm": "^10.0.0",
     "node": "^20.0.0"
   },
-  "keywords": ["postgres", "firebase", "storage", "functions", "database", "auth"],
-  "packageManager": "npm@10.7.0"
+  "keywords": [
+    "postgres",
+    "firebase",
+    "storage",
+    "functions",
+    "database",
+    "auth"
+  ],
+  "packageManager": "npm@10.7.0",
+  "dependencies": {
+    "@types/cheerio": "^0.22.35"
+  }
 }

packages/ai-commands/src/sql/chat.ts

@@ -14,7 +14,8 @@ export async function chatSql(
   messages: Message[],
   existingSql?: string,
   entityDefinitions?: string[],
-  context?: 'rls-policies' | 'functions'
+  context?: 'rls-policies' | 'functions',
+  webContent?: string
 ): Promise<ReadableStream<Uint8Array>> {
   const initMessages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
     {
@@ -82,6 +83,13 @@ export async function chatSql(
     })
   }
 
+  if (webContent) {
+    initMessages.push({
+      role: 'user',
+      content: `Here is some additional web content for context: ${webContent}`,
+    })
+  }
+
   if (messages) {
     initMessages.push(...messages)
   }