From 377637ca67eee95b5ece0682c038316bd6678062 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 6 Apr 2026 22:30:19 +0100 Subject: [PATCH] perf(secrets): load bundled web providers from public artifacts --- extensions/firecrawl/web-fetch-provider.ts | 1 + src/plugins/web-provider-public-artifacts.ts | 255 +++++++++++++++++++ src/secrets/runtime-web-tools.test.ts | 40 +++ src/secrets/runtime-web-tools.ts | 137 +++++++--- 4 files changed, 399 insertions(+), 34 deletions(-) create mode 100644 extensions/firecrawl/web-fetch-provider.ts create mode 100644 src/plugins/web-provider-public-artifacts.ts diff --git a/extensions/firecrawl/web-fetch-provider.ts b/extensions/firecrawl/web-fetch-provider.ts new file mode 100644 index 0000000000..800d1cdd9a --- /dev/null +++ b/extensions/firecrawl/web-fetch-provider.ts @@ -0,0 +1 @@ +export { createFirecrawlWebFetchProvider } from "./src/firecrawl-fetch-provider.js"; diff --git a/src/plugins/web-provider-public-artifacts.ts b/src/plugins/web-provider-public-artifacts.ts new file mode 100644 index 0000000000..1dbf3e6fd9 --- /dev/null +++ b/src/plugins/web-provider-public-artifacts.ts @@ -0,0 +1,255 @@ +import path from "node:path"; +import type { PluginLoadOptions } from "./loader.js"; +import { loadPluginManifestRegistry } from "./manifest-registry.js"; +import { loadBundledPluginPublicArtifactModuleSync } from "./public-surface-loader.js"; +import type { + PluginWebFetchProviderEntry, + PluginWebSearchProviderEntry, + WebFetchProviderPlugin, + WebSearchProviderPlugin, +} from "./types.js"; +import { resolveBundledWebFetchResolutionConfig } from "./web-fetch-providers.shared.js"; +import { resolveManifestDeclaredWebProviderCandidatePluginIds } from "./web-provider-resolution-shared.js"; +import { resolveBundledWebSearchResolutionConfig } from "./web-search-providers.shared.js"; + +const WEB_SEARCH_ARTIFACT_CANDIDATES = ["web-search-provider.js", "web-search.js"] as const; +const WEB_FETCH_ARTIFACT_CANDIDATES = ["web-fetch-provider.js", "web-fetch.js"] as const; + +type BundledWebProviderPublicArtifactParams = { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + onlyPluginIds?: readonly string[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string"); +} + +function isWebSearchProviderPlugin(value: unknown): value is WebSearchProviderPlugin { + return ( + isRecord(value) && + typeof value.id === "string" && + typeof value.label === "string" && + typeof value.hint === "string" && + isStringArray(value.envVars) && + typeof value.placeholder === "string" && + typeof value.signupUrl === "string" && + typeof value.credentialPath === "string" && + typeof value.getCredentialValue === "function" && + typeof value.setCredentialValue === "function" && + typeof value.createTool === "function" + ); +} + +function isWebFetchProviderPlugin(value: unknown): value is WebFetchProviderPlugin { + return ( + isRecord(value) && + typeof value.id === "string" && + typeof value.label === "string" && + typeof value.hint === "string" && + isStringArray(value.envVars) && + typeof value.placeholder === "string" && + typeof value.signupUrl === "string" && + typeof value.credentialPath === "string" && + typeof value.getCredentialValue === "function" && + typeof value.setCredentialValue === "function" && + typeof value.createTool === "function" + ); +} + +function collectProviderFactories(params: { + mod: Record; + suffix: string; + isProvider: (value: unknown) => value is TProvider; +}): TProvider[] { + const providers: TProvider[] = []; + for (const [name, exported] of Object.entries(params.mod).toSorted(([left], [right]) => + left.localeCompare(right), + )) { + if ( + typeof exported !== "function" || + exported.length !== 0 || + !name.startsWith("create") || + !name.endsWith(params.suffix) + ) { + continue; + } + const candidate = exported(); + if (params.isProvider(candidate)) { + providers.push(candidate); + } + } + return providers; +} + +function tryLoadBundledPublicArtifactModule(params: { + dirName: string; + artifactCandidates: readonly string[]; +}): Record | null { + for (const artifactBasename of params.artifactCandidates) { + try { + return loadBundledPluginPublicArtifactModuleSync>({ + dirName: params.dirName, + artifactBasename, + }); + } catch (error) { + if ( + error instanceof Error && + error.message.startsWith("Unable to resolve bundled plugin public surface ") + ) { + continue; + } + throw error; + } + } + return null; +} + +function resolveBundledCandidatePluginIds(params: { + contract: "webSearchProviders" | "webFetchProviders"; + configKey: "webSearch" | "webFetch"; + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + onlyPluginIds?: readonly string[]; +}): string[] { + if (params.onlyPluginIds && params.onlyPluginIds.length > 0) { + return [...new Set(params.onlyPluginIds)].toSorted((left, right) => left.localeCompare(right)); + } + const resolvedConfig = + params.contract === "webSearchProviders" + ? resolveBundledWebSearchResolutionConfig(params).config + : resolveBundledWebFetchResolutionConfig(params).config; + return ( + resolveManifestDeclaredWebProviderCandidatePluginIds({ + contract: params.contract, + configKey: params.configKey, + config: resolvedConfig, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: params.onlyPluginIds, + origin: "bundled", + }) ?? [] + ); +} + +function resolveBundledManifestRecordsByPluginId(params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + onlyPluginIds: readonly string[]; +}) { + const allowedPluginIds = new Set(params.onlyPluginIds); + return new Map( + loadPluginManifestRegistry({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + }) + .plugins.filter((record) => record.origin === "bundled" && allowedPluginIds.has(record.id)) + .map((record) => [record.id, record] as const), + ); +} + +export function resolveBundledWebSearchProvidersFromPublicArtifacts( + params: BundledWebProviderPublicArtifactParams, +): PluginWebSearchProviderEntry[] { + const pluginIds = resolveBundledCandidatePluginIds({ + contract: "webSearchProviders", + configKey: "webSearch", + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + bundledAllowlistCompat: params.bundledAllowlistCompat, + onlyPluginIds: params.onlyPluginIds, + }); + if (pluginIds.length === 0) { + return []; + } + const recordsByPluginId = resolveBundledManifestRecordsByPluginId({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: pluginIds, + }); + const providers: PluginWebSearchProviderEntry[] = []; + for (const pluginId of pluginIds) { + const record = recordsByPluginId.get(pluginId); + if (!record) { + continue; + } + const mod = tryLoadBundledPublicArtifactModule({ + dirName: path.basename(record.rootDir), + artifactCandidates: WEB_SEARCH_ARTIFACT_CANDIDATES, + }); + if (!mod) { + continue; + } + providers.push( + ...collectProviderFactories({ + mod, + suffix: "WebSearchProvider", + isProvider: isWebSearchProviderPlugin, + }).map((provider) => ({ + ...provider, + pluginId, + })), + ); + } + return providers; +} + +export function resolveBundledWebFetchProvidersFromPublicArtifacts( + params: BundledWebProviderPublicArtifactParams, +): PluginWebFetchProviderEntry[] { + const pluginIds = resolveBundledCandidatePluginIds({ + contract: "webFetchProviders", + configKey: "webFetch", + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + bundledAllowlistCompat: params.bundledAllowlistCompat, + onlyPluginIds: params.onlyPluginIds, + }); + if (pluginIds.length === 0) { + return []; + } + const recordsByPluginId = resolveBundledManifestRecordsByPluginId({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: pluginIds, + }); + const providers: PluginWebFetchProviderEntry[] = []; + for (const pluginId of pluginIds) { + const record = recordsByPluginId.get(pluginId); + if (!record) { + continue; + } + const mod = tryLoadBundledPublicArtifactModule({ + dirName: path.basename(record.rootDir), + artifactCandidates: WEB_FETCH_ARTIFACT_CANDIDATES, + }); + if (!mod) { + continue; + } + providers.push( + ...collectProviderFactories({ + mod, + suffix: "WebFetchProvider", + isProvider: isWebFetchProviderPlugin, + }).map((provider) => ({ + ...provider, + pluginId, + })), + ); + } + return providers; +} diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts index d6e8a95c99..234c980280 100644 --- a/src/secrets/runtime-web-tools.test.ts +++ b/src/secrets/runtime-web-tools.test.ts @@ -855,6 +855,46 @@ describe("runtime web tools resolution", () => { expect(runtimeWebFetchProviders.resolvePluginWebFetchProviders).not.toHaveBeenCalled(); }); + it("uses bundled public artifacts for bundled web search provider discovery", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + provider: "brave", + }, + }, + }, + }), + env: { + BRAVE_API_KEY: "brave-key", // pragma: allowlist secret + }, + }); + + expect(metadata.search.selectedProvider).toBe("brave"); + expect(runtimeWebSearchProviders.resolvePluginWebSearchProviders).not.toHaveBeenCalled(); + }); + + it("uses bundled public artifacts for bundled web fetch provider discovery", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret + }, + }); + + expect(metadata.fetch.selectedProvider).toBe("firecrawl"); + expect(runtimeWebFetchProviders.resolvePluginWebFetchProviders).not.toHaveBeenCalled(); + }); + it("uses env fallback for unresolved web fetch provider SecretRef when active", async () => { const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ config: asConfig({ diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index b7a1f0c664..613be20b1d 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -9,6 +9,10 @@ import type { } from "../plugins/types.js"; import { resolvePluginWebFetchProviders } from "../plugins/web-fetch-providers.runtime.js"; import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js"; +import { + resolveBundledWebFetchProvidersFromPublicArtifacts, + resolveBundledWebSearchProvidersFromPublicArtifacts, +} from "../plugins/web-provider-public-artifacts.js"; import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.runtime.js"; import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-providers.shared.js"; import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; @@ -254,6 +258,94 @@ function setResolvedWebSearchApiKey(params: { params.provider.setCredentialValue(search, params.value); } +function resolveBundledWebSearchProviders(params: { + sourceConfig: OpenClawConfig; + context: ResolverContext; + configuredBundledPluginId?: string; + hasCustomWebSearchPluginRisk: boolean; +}): PluginWebSearchProviderEntry[] { + const env = { ...process.env, ...params.context.env }; + if (params.configuredBundledPluginId) { + const bundled = resolveBundledWebSearchProvidersFromPublicArtifacts({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + onlyPluginIds: [params.configuredBundledPluginId], + }); + if (bundled.length > 0) { + return bundled; + } + return resolvePluginWebSearchProviders({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + onlyPluginIds: [params.configuredBundledPluginId], + origin: "bundled", + }); + } + if (!params.hasCustomWebSearchPluginRisk) { + const bundled = resolveBundledWebSearchProvidersFromPublicArtifacts({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + }); + if (bundled.length > 0) { + return bundled; + } + return resolvePluginWebSearchProviders({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + origin: "bundled", + }); + } + return resolvePluginWebSearchProviders({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + }); +} + +function resolveBundledWebFetchProviders(params: { + sourceConfig: OpenClawConfig; + context: ResolverContext; + configuredBundledPluginId?: string; +}): PluginWebFetchProviderEntry[] { + const env = { ...process.env, ...params.context.env }; + if (params.configuredBundledPluginId) { + const bundled = resolveBundledWebFetchProvidersFromPublicArtifacts({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + onlyPluginIds: [params.configuredBundledPluginId], + }); + if (bundled.length > 0) { + return bundled; + } + return resolvePluginWebFetchProviders({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + onlyPluginIds: [params.configuredBundledPluginId], + origin: "bundled", + }); + } + const bundled = resolveBundledWebFetchProvidersFromPublicArtifacts({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + }); + if (bundled.length > 0) { + return bundled; + } + return resolvePluginWebFetchProviders({ + config: params.sourceConfig, + env, + bundledAllowlistCompat: true, + origin: "bundled", + }); +} + function readConfiguredProviderCredential(params: { provider: PluginWebSearchProviderEntry; config: OpenClawConfig; @@ -390,26 +482,12 @@ export async function resolveRuntimeWebTools(params: { sourceConfig: params.sourceConfig, context: params.context, resolveProviders: ({ configuredBundledPluginId }) => - configuredBundledPluginId - ? resolvePluginWebSearchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - onlyPluginIds: [configuredBundledPluginId], - origin: "bundled", - }) - : !hasCustomWebSearchPluginRisk(params.sourceConfig) - ? resolvePluginWebSearchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - origin: "bundled", - }) - : resolvePluginWebSearchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - }), + resolveBundledWebSearchProviders({ + sourceConfig: params.sourceConfig, + context: params.context, + configuredBundledPluginId, + hasCustomWebSearchPluginRisk: hasCustomWebSearchPluginRisk(params.sourceConfig), + }), sortProviders: sortWebSearchProvidersForAutoDetect, readConfiguredCredential: ({ provider, config, toolConfig }) => readConfiguredProviderCredential({ @@ -500,20 +578,11 @@ export async function resolveRuntimeWebTools(params: { sourceConfig: params.sourceConfig, context: params.context, resolveProviders: ({ configuredBundledPluginId }) => - configuredBundledPluginId - ? resolvePluginWebFetchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - onlyPluginIds: [configuredBundledPluginId], - origin: "bundled", - }) - : resolvePluginWebFetchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - origin: "bundled", - }), + resolveBundledWebFetchProviders({ + sourceConfig: params.sourceConfig, + context: params.context, + configuredBundledPluginId, + }), sortProviders: sortWebFetchProvidersForAutoDetect, readConfiguredCredential: ({ provider, config, toolConfig }) => readConfiguredFetchProviderCredential({