perf(secrets): load bundled web providers from public artifacts
This commit is contained in:
Vincent Koc
@@ -0,0 +1 @@
|
||||
export { createFirecrawlWebFetchProvider } from "./src/firecrawl-fetch-provider.js";
|
||||
@@ -0,0 +1,255 @@
|
||||
import path from "node:path";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import { loadPluginManifestRegistry } from "./manifest-registry.js";
|
||||
import { loadBundledPluginPublicArtifactModuleSync } from "./public-surface-loader.js";
|
||||
import type {
|
||||
PluginWebFetchProviderEntry,
|
||||
PluginWebSearchProviderEntry,
|
||||
WebFetchProviderPlugin,
|
||||
WebSearchProviderPlugin,
|
||||
} from "./types.js";
|
||||
import { resolveBundledWebFetchResolutionConfig } from "./web-fetch-providers.shared.js";
|
||||
import { resolveManifestDeclaredWebProviderCandidatePluginIds } from "./web-provider-resolution-shared.js";
|
||||
import { resolveBundledWebSearchResolutionConfig } from "./web-search-providers.shared.js";
|
||||
|
||||
const WEB_SEARCH_ARTIFACT_CANDIDATES = ["web-search-provider.js", "web-search.js"] as const;
|
||||
const WEB_FETCH_ARTIFACT_CANDIDATES = ["web-fetch-provider.js", "web-fetch.js"] as const;
|
||||
|
||||
type BundledWebProviderPublicArtifactParams = {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isStringArray(value: unknown): value is string[] {
|
||||
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
||||
}
|
||||
|
||||
function isWebSearchProviderPlugin(value: unknown): value is WebSearchProviderPlugin {
|
||||
return (
|
||||
isRecord(value) &&
|
||||
typeof value.id === "string" &&
|
||||
typeof value.label === "string" &&
|
||||
typeof value.hint === "string" &&
|
||||
isStringArray(value.envVars) &&
|
||||
typeof value.placeholder === "string" &&
|
||||
typeof value.signupUrl === "string" &&
|
||||
typeof value.credentialPath === "string" &&
|
||||
typeof value.getCredentialValue === "function" &&
|
||||
typeof value.setCredentialValue === "function" &&
|
||||
typeof value.createTool === "function"
|
||||
);
|
||||
}
|
||||
|
||||
function isWebFetchProviderPlugin(value: unknown): value is WebFetchProviderPlugin {
|
||||
return (
|
||||
isRecord(value) &&
|
||||
typeof value.id === "string" &&
|
||||
typeof value.label === "string" &&
|
||||
typeof value.hint === "string" &&
|
||||
isStringArray(value.envVars) &&
|
||||
typeof value.placeholder === "string" &&
|
||||
typeof value.signupUrl === "string" &&
|
||||
typeof value.credentialPath === "string" &&
|
||||
typeof value.getCredentialValue === "function" &&
|
||||
typeof value.setCredentialValue === "function" &&
|
||||
typeof value.createTool === "function"
|
||||
);
|
||||
}
|
||||
|
||||
function collectProviderFactories<TProvider>(params: {
|
||||
mod: Record<string, unknown>;
|
||||
suffix: string;
|
||||
isProvider: (value: unknown) => value is TProvider;
|
||||
}): TProvider[] {
|
||||
const providers: TProvider[] = [];
|
||||
for (const [name, exported] of Object.entries(params.mod).toSorted(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
)) {
|
||||
if (
|
||||
typeof exported !== "function" ||
|
||||
exported.length !== 0 ||
|
||||
!name.startsWith("create") ||
|
||||
!name.endsWith(params.suffix)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const candidate = exported();
|
||||
if (params.isProvider(candidate)) {
|
||||
providers.push(candidate);
|
||||
}
|
||||
}
|
||||
return providers;
|
||||
}
|
||||
|
||||
function tryLoadBundledPublicArtifactModule(params: {
|
||||
dirName: string;
|
||||
artifactCandidates: readonly string[];
|
||||
}): Record<string, unknown> | null {
|
||||
for (const artifactBasename of params.artifactCandidates) {
|
||||
try {
|
||||
return loadBundledPluginPublicArtifactModuleSync<Record<string, unknown>>({
|
||||
dirName: params.dirName,
|
||||
artifactBasename,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.startsWith("Unable to resolve bundled plugin public surface ")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveBundledCandidatePluginIds(params: {
|
||||
contract: "webSearchProviders" | "webFetchProviders";
|
||||
configKey: "webSearch" | "webFetch";
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
}): string[] {
|
||||
if (params.onlyPluginIds && params.onlyPluginIds.length > 0) {
|
||||
return [...new Set(params.onlyPluginIds)].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
const resolvedConfig =
|
||||
params.contract === "webSearchProviders"
|
||||
? resolveBundledWebSearchResolutionConfig(params).config
|
||||
: resolveBundledWebFetchResolutionConfig(params).config;
|
||||
return (
|
||||
resolveManifestDeclaredWebProviderCandidatePluginIds({
|
||||
contract: params.contract,
|
||||
configKey: params.configKey,
|
||||
config: resolvedConfig,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
origin: "bundled",
|
||||
}) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
function resolveBundledManifestRecordsByPluginId(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
onlyPluginIds: readonly string[];
|
||||
}) {
|
||||
const allowedPluginIds = new Set(params.onlyPluginIds);
|
||||
return new Map(
|
||||
loadPluginManifestRegistry({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
})
|
||||
.plugins.filter((record) => record.origin === "bundled" && allowedPluginIds.has(record.id))
|
||||
.map((record) => [record.id, record] as const),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveBundledWebSearchProvidersFromPublicArtifacts(
|
||||
params: BundledWebProviderPublicArtifactParams,
|
||||
): PluginWebSearchProviderEntry[] {
|
||||
const pluginIds = resolveBundledCandidatePluginIds({
|
||||
contract: "webSearchProviders",
|
||||
configKey: "webSearch",
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
bundledAllowlistCompat: params.bundledAllowlistCompat,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
});
|
||||
if (pluginIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const recordsByPluginId = resolveBundledManifestRecordsByPluginId({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
onlyPluginIds: pluginIds,
|
||||
});
|
||||
const providers: PluginWebSearchProviderEntry[] = [];
|
||||
for (const pluginId of pluginIds) {
|
||||
const record = recordsByPluginId.get(pluginId);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
const mod = tryLoadBundledPublicArtifactModule({
|
||||
dirName: path.basename(record.rootDir),
|
||||
artifactCandidates: WEB_SEARCH_ARTIFACT_CANDIDATES,
|
||||
});
|
||||
if (!mod) {
|
||||
continue;
|
||||
}
|
||||
providers.push(
|
||||
...collectProviderFactories({
|
||||
mod,
|
||||
suffix: "WebSearchProvider",
|
||||
isProvider: isWebSearchProviderPlugin,
|
||||
}).map((provider) => ({
|
||||
...provider,
|
||||
pluginId,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return providers;
|
||||
}
|
||||
|
||||
export function resolveBundledWebFetchProvidersFromPublicArtifacts(
|
||||
params: BundledWebProviderPublicArtifactParams,
|
||||
): PluginWebFetchProviderEntry[] {
|
||||
const pluginIds = resolveBundledCandidatePluginIds({
|
||||
contract: "webFetchProviders",
|
||||
configKey: "webFetch",
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
bundledAllowlistCompat: params.bundledAllowlistCompat,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
});
|
||||
if (pluginIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const recordsByPluginId = resolveBundledManifestRecordsByPluginId({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
onlyPluginIds: pluginIds,
|
||||
});
|
||||
const providers: PluginWebFetchProviderEntry[] = [];
|
||||
for (const pluginId of pluginIds) {
|
||||
const record = recordsByPluginId.get(pluginId);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
const mod = tryLoadBundledPublicArtifactModule({
|
||||
dirName: path.basename(record.rootDir),
|
||||
artifactCandidates: WEB_FETCH_ARTIFACT_CANDIDATES,
|
||||
});
|
||||
if (!mod) {
|
||||
continue;
|
||||
}
|
||||
providers.push(
|
||||
...collectProviderFactories({
|
||||
mod,
|
||||
suffix: "WebFetchProvider",
|
||||
isProvider: isWebFetchProviderPlugin,
|
||||
}).map((provider) => ({
|
||||
...provider,
|
||||
pluginId,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return providers;
|
||||
}
|
||||
@@ -855,6 +855,46 @@ describe("runtime web tools resolution", () => {
|
||||
expect(runtimeWebFetchProviders.resolvePluginWebFetchProviders).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses bundled public artifacts for bundled web search provider discovery", async () => {
|
||||
const { metadata } = await runRuntimeWebTools({
|
||||
config: asConfig({
|
||||
tools: {
|
||||
web: {
|
||||
search: {
|
||||
provider: "brave",
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: {
|
||||
BRAVE_API_KEY: "brave-key", // pragma: allowlist secret
|
||||
},
|
||||
});
|
||||
|
||||
expect(metadata.search.selectedProvider).toBe("brave");
|
||||
expect(runtimeWebSearchProviders.resolvePluginWebSearchProviders).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses bundled public artifacts for bundled web fetch provider discovery", async () => {
|
||||
const { metadata } = await runRuntimeWebTools({
|
||||
config: asConfig({
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: "firecrawl",
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: {
|
||||
FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret
|
||||
},
|
||||
});
|
||||
|
||||
expect(metadata.fetch.selectedProvider).toBe("firecrawl");
|
||||
expect(runtimeWebFetchProviders.resolvePluginWebFetchProviders).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses env fallback for unresolved web fetch provider SecretRef when active", async () => {
|
||||
const { metadata, resolvedConfig, context } = await runRuntimeWebTools({
|
||||
config: asConfig({
|
||||
|
||||
@@ -9,6 +9,10 @@ import type {
|
||||
} from "../plugins/types.js";
|
||||
import { resolvePluginWebFetchProviders } from "../plugins/web-fetch-providers.runtime.js";
|
||||
import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js";
|
||||
import {
|
||||
resolveBundledWebFetchProvidersFromPublicArtifacts,
|
||||
resolveBundledWebSearchProvidersFromPublicArtifacts,
|
||||
} from "../plugins/web-provider-public-artifacts.js";
|
||||
import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.runtime.js";
|
||||
import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-providers.shared.js";
|
||||
import { normalizeSecretInput } from "../utils/normalize-secret-input.js";
|
||||
@@ -254,6 +258,94 @@ function setResolvedWebSearchApiKey(params: {
|
||||
params.provider.setCredentialValue(search, params.value);
|
||||
}
|
||||
|
||||
function resolveBundledWebSearchProviders(params: {
|
||||
sourceConfig: OpenClawConfig;
|
||||
context: ResolverContext;
|
||||
configuredBundledPluginId?: string;
|
||||
hasCustomWebSearchPluginRisk: boolean;
|
||||
}): PluginWebSearchProviderEntry[] {
|
||||
const env = { ...process.env, ...params.context.env };
|
||||
if (params.configuredBundledPluginId) {
|
||||
const bundled = resolveBundledWebSearchProvidersFromPublicArtifacts({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [params.configuredBundledPluginId],
|
||||
});
|
||||
if (bundled.length > 0) {
|
||||
return bundled;
|
||||
}
|
||||
return resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [params.configuredBundledPluginId],
|
||||
origin: "bundled",
|
||||
});
|
||||
}
|
||||
if (!params.hasCustomWebSearchPluginRisk) {
|
||||
const bundled = resolveBundledWebSearchProvidersFromPublicArtifacts({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
});
|
||||
if (bundled.length > 0) {
|
||||
return bundled;
|
||||
}
|
||||
return resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
origin: "bundled",
|
||||
});
|
||||
}
|
||||
return resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveBundledWebFetchProviders(params: {
|
||||
sourceConfig: OpenClawConfig;
|
||||
context: ResolverContext;
|
||||
configuredBundledPluginId?: string;
|
||||
}): PluginWebFetchProviderEntry[] {
|
||||
const env = { ...process.env, ...params.context.env };
|
||||
if (params.configuredBundledPluginId) {
|
||||
const bundled = resolveBundledWebFetchProvidersFromPublicArtifacts({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [params.configuredBundledPluginId],
|
||||
});
|
||||
if (bundled.length > 0) {
|
||||
return bundled;
|
||||
}
|
||||
return resolvePluginWebFetchProviders({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [params.configuredBundledPluginId],
|
||||
origin: "bundled",
|
||||
});
|
||||
}
|
||||
const bundled = resolveBundledWebFetchProvidersFromPublicArtifacts({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
});
|
||||
if (bundled.length > 0) {
|
||||
return bundled;
|
||||
}
|
||||
return resolvePluginWebFetchProviders({
|
||||
config: params.sourceConfig,
|
||||
env,
|
||||
bundledAllowlistCompat: true,
|
||||
origin: "bundled",
|
||||
});
|
||||
}
|
||||
|
||||
function readConfiguredProviderCredential(params: {
|
||||
provider: PluginWebSearchProviderEntry;
|
||||
config: OpenClawConfig;
|
||||
@@ -390,26 +482,12 @@ export async function resolveRuntimeWebTools(params: {
|
||||
sourceConfig: params.sourceConfig,
|
||||
context: params.context,
|
||||
resolveProviders: ({ configuredBundledPluginId }) =>
|
||||
configuredBundledPluginId
|
||||
? resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env: { ...process.env, ...params.context.env },
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [configuredBundledPluginId],
|
||||
origin: "bundled",
|
||||
})
|
||||
: !hasCustomWebSearchPluginRisk(params.sourceConfig)
|
||||
? resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env: { ...process.env, ...params.context.env },
|
||||
bundledAllowlistCompat: true,
|
||||
origin: "bundled",
|
||||
})
|
||||
: resolvePluginWebSearchProviders({
|
||||
config: params.sourceConfig,
|
||||
env: { ...process.env, ...params.context.env },
|
||||
bundledAllowlistCompat: true,
|
||||
}),
|
||||
resolveBundledWebSearchProviders({
|
||||
sourceConfig: params.sourceConfig,
|
||||
context: params.context,
|
||||
configuredBundledPluginId,
|
||||
hasCustomWebSearchPluginRisk: hasCustomWebSearchPluginRisk(params.sourceConfig),
|
||||
}),
|
||||
sortProviders: sortWebSearchProvidersForAutoDetect,
|
||||
readConfiguredCredential: ({ provider, config, toolConfig }) =>
|
||||
readConfiguredProviderCredential({
|
||||
@@ -500,20 +578,11 @@ export async function resolveRuntimeWebTools(params: {
|
||||
sourceConfig: params.sourceConfig,
|
||||
context: params.context,
|
||||
resolveProviders: ({ configuredBundledPluginId }) =>
|
||||
configuredBundledPluginId
|
||||
? resolvePluginWebFetchProviders({
|
||||
config: params.sourceConfig,
|
||||
env: { ...process.env, ...params.context.env },
|
||||
bundledAllowlistCompat: true,
|
||||
onlyPluginIds: [configuredBundledPluginId],
|
||||
origin: "bundled",
|
||||
})
|
||||
: resolvePluginWebFetchProviders({
|
||||
config: params.sourceConfig,
|
||||
env: { ...process.env, ...params.context.env },
|
||||
bundledAllowlistCompat: true,
|
||||
origin: "bundled",
|
||||
}),
|
||||
resolveBundledWebFetchProviders({
|
||||
sourceConfig: params.sourceConfig,
|
||||
context: params.context,
|
||||
configuredBundledPluginId,
|
||||
}),
|
||||
sortProviders: sortWebFetchProvidersForAutoDetect,
|
||||
readConfiguredCredential: ({ provider, config, toolConfig }) =>
|
||||
readConfiguredFetchProviderCredential({
|
||||
|
||||
Reference in New Issue
Block a user