fix: search and sync scripts (#38756)

* fix: search script

* fix: sync script
This commit is contained in:
Charis authored and GitHub committed 2025-09-17 11:20:43 -04:00
1 parent 506b20734f
commit fd308c6bad
2 files changed
+17 -19

No files matched your search

@@ -4,7 +4,7 @@ import { Result } from '~/features/helpers.fn'
import { openAI } from '~/lib/openAi'
import { supabase, type DatabaseCorrected } from '~/lib/supabase'
import { isFeatureEnabled } from 'common/enabled-features'
import { isFeatureEnabled } from '../../../../packages/common/enabled-features'
import { GuideModel } from '../guide/guideModel'
import {
DB_METADATA_TAG_PLATFORM_CLI,
+16 -18
View File
@@ -178,12 +178,14 @@ async function prepareSections(
return
}
if (existingPage && debug) {
console.log(
!shouldRefresh
? `[${path}] Docs have changed, removing old page sections and their embeddings`
: `[${path}] Refresh flag set, removing old page sections and their embeddings`
)
if (existingPage) {
if (debug) {
console.log(
!shouldRefresh
? `[${path}] Docs have changed, removing old page sections and their embeddings`
: `[${path}] Refresh flag set, removing old page sections and their embeddings`
)
}
const { error: deletePageSectionError } = await supabaseClient
.from(pageSectionTable)
@@ -254,18 +256,6 @@ async function processAndInsertEmbeddings(
allSections: PageSectionForEmbedding[],
pageInfoMap: Map<number, PageInfo>
): Promise<ProcessingResult> {
if (allSections.length === 0) {
return {
successfulPages: new Set(),
failedPages: new Set(),
totalSectionsProcessed: 0,
totalSectionsInserted: 0,
}
}
console.log(`Processing ${allSections.length} sections with embeddings + insertion`)
const embeddingBatches = createBatches(allSections, CONFIG.OPENAI_BATCH_SIZE)
const result: ProcessingResult = {
successfulPages: new Set(),
failedPages: new Set(),
@@ -273,6 +263,14 @@ async function processAndInsertEmbeddings(
totalSectionsInserted: 0,
}
if (allSections.length === 0) {
return result
}
console.log(`Processing ${allSections.length} sections with embeddings + insertion`)
const embeddingBatches = createBatches(allSections, CONFIG.OPENAI_BATCH_SIZE)
// Track sections inserted per page
const pageSectionsInserted = new Map<number, number>()