From db5e6fc50fcb265072e3c40c35ca92bfe5526587 Mon Sep 17 00:00:00 2001 From: Terry Sutton Date: Wed, 23 Oct 2024 21:10:07 -0230 Subject: [PATCH] Add option to add web content --- .../ui/AIAssistantPanel/AIAssistant.tsx | 76 ++++++++++- apps/studio/pages/api/ai/sql/generate-v2.ts | 13 +- .../pages/api/ai/web-url/get-url-content.ts | 121 ++++++++++++++++++ package-lock.json | 11 ++ package.json | 21 ++- packages/ai-commands/src/sql/chat.ts | 10 +- 6 files changed, 243 insertions(+), 9 deletions(-) create mode 100644 apps/studio/pages/api/ai/web-url/get-url-content.ts diff --git a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx index 90a008b41b0..b58171cd6f6 100644 --- a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx +++ b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx @@ -3,7 +3,7 @@ import { AnimatePresence, motion } from 'framer-motion' import { last } from 'lodash' import { ExternalLink, FileText, MessageCircleMore, Plus, WandSparkles } from 'lucide-react' import Link from 'next/link' -import { useEffect, useMemo, useRef, useState } from 'react' +import { FormEvent, useEffect, useMemo, useRef, useState } from 'react' import { toast } from 'sonner' import type { Message as MessageType } from 'ai/react' @@ -38,6 +38,7 @@ import { DropdownMenuSubContent, DropdownMenuSubTrigger, DropdownMenuTrigger, + Input_Shadcn_, SheetHeader, SheetSection, Tooltip_Shadcn_, @@ -98,6 +99,8 @@ export const AIAssistant = ({ const [assistantError, setAssistantError] = useState() const [lastSentMessage, setLastSentMessage] = useState() const [isConfirmOptInModalOpen, setIsConfirmOptInModalOpen] = useState(false) + const [webUrl, setWebUrl] = useState('') + const [webContent, setWebContent] = useState('') const docsUrl = retrieveDocsUrl(selectedDatabaseEntity as SupportedAssistantEntities) const entityContext = ASSISTANT_SUPPORT_ENTITIES.find((x) => x.id === selectedDatabaseEntity) @@ -134,7 +137,7 @@ export const AIAssistant = ({ } = useChat({ id, api: `${BASE_PATH}/api/ai/sql/generate-v2`, - body: { entityDefinitions, context: selectedDatabaseEntity }, + body: { entityDefinitions, context: selectedDatabaseEntity, webContent }, onError: (error) => setAssistantError(JSON.parse(error.message).error), }) @@ -275,6 +278,35 @@ export const AIAssistant = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [messages.length]) + async function handleWebSubmit(e: FormEvent) { + e.preventDefault() + + if (!webUrl) return + + try { + const response = await fetch(`${BASE_PATH}/api/ai/web-url/get-url-content`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ url: webUrl }), + }) + + if (!response.ok) { + throw new Error('Failed to fetch web content') + } + + const data = await response.json() + console.log('Fetched web content:', data.content) + + setWebContent(data.content) + } catch (error) { + console.error('Error fetching web content:', error) + toast.error('Failed to fetch web content. Please try again.') + } + + // setWebUrl('') + } return ( <>
@@ -508,6 +540,27 @@ export const AIAssistant = ({ )} + + +
+

Web

+

+ Add a link to a website to help the assistant answer your question +

+
+ + +
+
+ setWebUrl(e.target.value)} + /> + +
+ + )} @@ -574,6 +627,25 @@ export const AIAssistant = ({ } /> )} + {webUrl.length > 0 && ( + 0 ? webContent.slice(0, 20) + '...' : webUrl} + onRemove={() => setWebUrl('')} + tooltip={ + webContent.length > 20 ? ( + <> +

+ {webContent.length} site selected: +

+
    +
  • {webUrl}
    • +
+ + ) : undefined + } + /> + )}
) - const { messages, existingSql, entityDefinitions, context } = body + const { messages, existingSql, entityDefinitions, context, webContent } = body try { - const stream = await chatSql(openai, messages, existingSql, entityDefinitions, context) + const stream = await chatSql( + openai, + messages, + existingSql, + entityDefinitions, + context, + webContent + ) return new StreamingTextResponse(stream) } catch (error) { if (error instanceof Error) { diff --git a/apps/studio/pages/api/ai/web-url/get-url-content.ts b/apps/studio/pages/api/ai/web-url/get-url-content.ts new file mode 100644 index 00000000000..ab846343590 --- /dev/null +++ b/apps/studio/pages/api/ai/web-url/get-url-content.ts @@ -0,0 +1,121 @@ +import { NextApiRequest, NextApiResponse } from 'next' +import apiWrapper from 'lib/api/apiWrapper' +import * as cheerio from 'cheerio' + +export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler) + +async function handler(req: NextApiRequest, res: NextApiResponse) { + if (req.method !== 'POST') { + return res.status(405).json({ error: { message: 'Method Not Allowed' } }) + } + + const { url } = req.body + + if (!url) { + return res.status(400).json({ error: { message: 'URL is required' } }) + } + + try { + const response = await fetch(url) + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`) + } + const html = await response.text() + const $ = cheerio.load(html) + + // Extract title + const title = $('title').text().trim() + + // Extract meta description + const description = $('meta[name="description"]').attr('content')?.trim() || '' + + // Extract main content + const mainContent = extractMainContent($) + + return res.status(200).json({ + title, + description, + content: mainContent, + url, + }) + } catch (error) { + console.error('Error fetching web content:', error) + return res.status(500).json({ error: { message: 'Failed to fetch web content' } }) + } +} + +function extractMainContent($: cheerio.CheerioAPI): string { + // Remove script tags, style tags, and comments + $('script, style, comment').remove() + + // Common selectors for main content + const contentSelectors = [ + 'article', + '[role="main"]', + '.main-content', + '#main-content', + '.post-content', + '.entry-content', + '.content', + '#content', + 'main', + ] + + let mainContent = '' + + // Try to find content using common selectors + for (const selector of contentSelectors) { + const element = $(selector) + if (element.length > 0) { + mainContent = element.text().trim() + break + } + } + + // If no content found, use a density-based approach + if (!mainContent) { + mainContent = findContentByDensity($) + } + + // Fallback to body content if still empty + if (!mainContent) { + mainContent = $('body').text().trim() + } + + // Limit content length and clean it up + return cleanAndLimitContent(mainContent) +} + +function findContentByDensity($: cheerio.CheerioAPI): string { + let bestElement: cheerio.Element | null = null + let maxScore = 0 + + $('p') + .parent() + .each((_, element) => { + const $element = $(element) + const text = $element.text() + const linkDensity = $element.find('a').text().length / text.length || 0 + const score = text.length * (1 - linkDensity) + + if (score > maxScore) { + maxScore = score + bestElement = element + } + }) + + return bestElement ? $(bestElement).text().trim() : '' +} + +function cleanAndLimitContent(content: string): string { + // Remove extra whitespace and newlines + content = content.replace(/\s+/g, ' ').trim() + + // Limit to around 1000 words + const words = content.split(' ') + if (words.length > 1000) { + content = words.slice(0, 1000).join(' ') + '...' + } + + return content +} diff --git a/package-lock.json b/package-lock.json index 2a1db90666b..900626bf269 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,9 @@ "playwright-tests", "packages/*" ], + "dependencies": { + "@types/cheerio": "^0.22.35" + }, "devDependencies": { "eslint": "^8.57.0", "prettier": "^4.0.0-alpha.8", @@ -16452,6 +16455,14 @@ "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.2.tgz", "integrity": "sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg==" }, + "node_modules/@types/cheerio": { + "version": "0.22.35", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz", + "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/classnames": { "version": "2.3.1", "dev": true, diff --git a/package.json b/package.json index 82a14eb6657..35e2ba52461 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,12 @@ "author": "Supabase, Inc.", "license": "Apache-2.0", "private": true, - "workspaces": ["apps/*", "tests", "playwright-tests", "packages/*"], + "workspaces": [ + "apps/*", + "tests", + "playwright-tests", + "packages/*" + ], "scripts": { "build": "turbo run build", "build:studio": "turbo run build --filter=studio", @@ -53,6 +58,16 @@ "npm": "^10.0.0", "node": "^20.0.0" }, - "keywords": ["postgres", "firebase", "storage", "functions", "database", "auth"], - "packageManager": "npm@10.7.0" + "keywords": [ + "postgres", + "firebase", + "storage", + "functions", + "database", + "auth" + ], + "packageManager": "npm@10.7.0", + "dependencies": { + "@types/cheerio": "^0.22.35" + } } diff --git a/packages/ai-commands/src/sql/chat.ts b/packages/ai-commands/src/sql/chat.ts index 4b8457cd74f..536e8aa47fa 100644 --- a/packages/ai-commands/src/sql/chat.ts +++ b/packages/ai-commands/src/sql/chat.ts @@ -14,7 +14,8 @@ export async function chatSql( messages: Message[], existingSql?: string, entityDefinitions?: string[], - context?: 'rls-policies' | 'functions' + context?: 'rls-policies' | 'functions', + webContent?: string ): Promise> { const initMessages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [ { @@ -82,6 +83,13 @@ export async function chatSql( }) } + if (webContent) { + initMessages.push({ + role: 'user', + content: `Here is some additional web content for context: ${webContent}`, + }) + } + if (messages) { initMessages.push(...messages) }